@hanhnd/agent-kit 1.0.38 → 1.0.39

package/dist/cli.js

@@ -15504,10 +15504,6 @@ var MemoryStore = class {
     }
     return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
   }
-  normalizeVectorDistance(distance) {
-    if (distance <= 0) return 1;
-    return Math.max(0, Math.min(1, 1 - distance));
-  }
   hashesBySource(source) {
     const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
     return new Set(rows.map((r) => r.id));
@@ -15544,61 +15540,12 @@ var MemoryStore = class {
     });
     deleteAll();
   }
-  searchDense(embedding, limit) {
-    try {
-      const queryVector = this.toVectorBinding(embedding);
-      const rows = this.db.prepare(
-        `SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
-           FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
-           JOIN memory_chunks mc ON mc.rowid = vector_matches.id
-           ORDER BY distance
-           LIMIT ?`
-      ).all(queryVector, queryVector, limit, limit);
-      return rows.map((r) => ({
-        id: r.id,
-        score: this.normalizeVectorDistance(r.distance)
-      }));
-    } catch (err) {
-      console.warn("[memory-store] Dense search failed:", err);
-      return [];
-    }
-  }
-  searchBm25(query, limit) {
-    if (!query.trim()) return [];
+  buildFtsQuery(query) {
     const tokens = query.trim().split(/\s+/).map((t) => t.replace(/[^a-zA-Z0-9_]/g, "").toLowerCase()).filter((t) => t.length > 1);
-    const ftsQuery = (0, import_stopword.removeStopwords)(tokens, import_stopword.eng).map((t) => `${t.replace(/s$/, "")}*`).join(" OR ");
-    if (!ftsQuery) return [];
-    try {
-      const rows = this.db.prepare(
-        `SELECT mc.id, bm25(memory_fts, 0.2, 2.0, 5.0) AS rank
-           FROM memory_fts
-           JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
-           WHERE memory_fts MATCH ?
-           ORDER BY rank
-           LIMIT ?`
-      ).all(ftsQuery, limit);
-      if (rows.length === 0) return [];
-      const ranks = rows.map((r) => r.rank);
-      const minRank = Math.min(...ranks);
-      const maxRank = Math.max(...ranks);
-      const range = maxRank - minRank;
-      return rows.map((r) => ({
-        id: r.id,
-        score: range > 0 ? (maxRank - r.rank) / range : 1
-      }));
-    } catch (err) {
-      console.warn("[memory-store] BM25 search failed:", err);
-      return [];
-    }
+    return (0, import_stopword.removeStopwords)(tokens, import_stopword.eng).map((t) => `${t.replace(/s$/, "")}*`).join(" OR ");
   }
-  getChunksByIds(ids) {
-    if (ids.length === 0) return [];
-    const placeholders = ids.map(() => "?").join(", ");
-    const rows = this.db.prepare(
-      `SELECT id, source, source_type, heading, heading_level, content, line_start, line_end, file_mtime_at
-         FROM memory_chunks WHERE id IN (${placeholders})`
-    ).all(...ids);
-    return rows.map((r) => ({
+  mapMemoryChunkRow(r) {
+    return {
       id: r.id,
       source: r.source,
       sourceType: r.source_type,
@@ -15608,7 +15555,201 @@ var MemoryStore = class {
       lineStart: r.line_start,
       lineEnd: r.line_end,
       fileMtimeAt: r.file_mtime_at
-    }));
+    };
+  }
+  searchHybrid(options) {
+    const {
+      query,
+      embedding,
+      topK,
+      fetchLimit,
+      denseScoreFloor,
+      recencyWeight,
+      rrfK,
+      sourceType,
+      sinceMtimeAt,
+      includeDebug
+    } = options;
+    const ftsQuery = this.buildFtsQuery(query);
+    const hasDense = embedding !== void 0;
+    const hasBm25 = ftsQuery.length > 0;
+    if (!hasDense && !hasBm25) return [];
+    const hasFilters = sourceType !== void 0 || sinceMtimeAt !== void 0;
+    try {
+      const params = [];
+      let denseCte = "";
+      let denseRankedCte = "";
+      if (hasDense) {
+        const queryBuf = this.toVectorBinding(embedding);
+        if (hasFilters) {
+          const filterClauses = [];
+          if (sourceType) {
+            filterClauses.push("mc.source_type = ?");
+            params.push(sourceType);
+          }
+          if (sinceMtimeAt !== void 0) {
+            filterClauses.push("mc.file_mtime_at >= ?");
+            params.push(sinceMtimeAt);
+          }
+          const whereClause = filterClauses.length > 0 ? `WHERE ${filterClauses.join(" AND ")}` : "";
+          params.push(queryBuf);
+          params.push(fetchLimit);
+          denseCte = `dense_data AS (
+            SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                   mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                   vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
+            FROM memory_chunks mc
+            ${whereClause}
+            ORDER BY dense_dist
+            LIMIT ?
+          )`;
+        } else {
+          params.push(queryBuf);
+          params.push(queryBuf);
+          params.push(fetchLimit);
+          denseCte = `dense_data AS (
+            SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                   mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                   vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
+            FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vtk
+            JOIN memory_chunks mc ON mc.rowid = vtk.id
+          )`;
+        }
+        denseRankedCte = `dense_ranked AS (
+          SELECT *, ROW_NUMBER() OVER (ORDER BY dense_dist, id) AS dense_rank
+          FROM dense_data
+        )`;
+      }
+      let bm25Cte = "";
+      let bm25RankedCte = "";
+      if (hasBm25) {
+        const bm25FilterClauses = [];
+        if (sourceType) bm25FilterClauses.push("mc.source_type = ?");
+        if (sinceMtimeAt !== void 0) bm25FilterClauses.push("mc.file_mtime_at >= ?");
+        const bm25WhereExtra = bm25FilterClauses.length > 0 ? ` AND ${bm25FilterClauses.join(" AND ")}` : "";
+        params.push(ftsQuery);
+        if (sourceType) params.push(sourceType);
+        if (sinceMtimeAt !== void 0) params.push(sinceMtimeAt);
+        params.push(fetchLimit);
+        bm25Cte = `bm25_data AS (
+          SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                 mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                 bm25(memory_fts, 0.2, 2.0, 5.0) AS bm25_neg
+          FROM memory_fts
+          JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
+          WHERE memory_fts MATCH ?${bm25WhereExtra}
+          LIMIT ?
+        )`;
+        bm25RankedCte = `bm25_ranked AS (
+          SELECT *, ROW_NUMBER() OVER (ORDER BY bm25_neg, id) AS bm25_rank
+          FROM bm25_data
+        )`;
+      }
+      let candidatesCte = "";
+      if (hasDense && hasBm25) {
+        candidatesCte = `all_candidates AS (
+          SELECT d.id, d.source, d.source_type, d.file_mtime_at,
+                 d.heading, d.heading_level, d.content, d.line_start, d.line_end,
+                 CAST(d.dense_rank AS INTEGER) AS dense_rank, d.dense_dist,
+                 b.bm25_rank,
+                 CASE WHEN b.id IS NOT NULL THEN 'both' ELSE 'dense' END AS retriever
+          FROM dense_ranked d
+          LEFT JOIN bm25_ranked b ON b.id = d.id
+          UNION ALL
+          SELECT b.id, b.source, b.source_type, b.file_mtime_at,
+                 b.heading, b.heading_level, b.content, b.line_start, b.line_end,
+                 NULL AS dense_rank, NULL AS dense_dist,
+                 CAST(b.bm25_rank AS INTEGER) AS bm25_rank,
+                 'bm25' AS retriever
+          FROM bm25_ranked b
+          LEFT JOIN dense_ranked d ON d.id = b.id
+          WHERE d.id IS NULL
+        )`;
+      } else if (hasDense) {
+        candidatesCte = `all_candidates AS (
+          SELECT id, source, source_type, file_mtime_at,
+                 heading, heading_level, content, line_start, line_end,
+                 CAST(dense_rank AS INTEGER) AS dense_rank, dense_dist,
+                 NULL AS bm25_rank, 'dense' AS retriever
+          FROM dense_ranked
+        )`;
+      } else {
+        candidatesCte = `all_candidates AS (
+          SELECT id, source, source_type, file_mtime_at,
+                 heading, heading_level, content, line_start, line_end,
+                 NULL AS dense_rank, NULL AS dense_dist,
+                 CAST(bm25_rank AS INTEGER) AS bm25_rank, 'bm25' AS retriever
+          FROM bm25_ranked
+        )`;
+      }
+      params.push(rrfK, rrfK, recencyWeight, rrfK, denseScoreFloor, topK);
+      const sql = `
+        WITH
+          ${[denseCte, denseRankedCte, bm25Cte, bm25RankedCte, candidatesCte].filter(Boolean).join(",\n")}
+        ,
+        recency_ranked AS (
+          SELECT id, ROW_NUMBER() OVER (ORDER BY file_mtime_at DESC, source, id) AS recency_rank
+          FROM all_candidates
+        ),
+        scored AS (
+          SELECT c.id, c.source, c.source_type, c.file_mtime_at,
+                 c.heading, c.heading_level, c.content, c.line_start, c.line_end,
+                 c.dense_rank, c.dense_dist, c.bm25_rank, r.recency_rank,
+                 c.retriever,
+                 COALESCE(1.0 / (? + c.dense_rank), 0.0) +
+                 COALESCE(1.0 / (? + c.bm25_rank), 0.0) +
+                 ? * (1.0 / (? + r.recency_rank)) AS total_score
+          FROM all_candidates c
+          JOIN recency_ranked r ON r.id = c.id
+          WHERE c.retriever != 'dense' OR (1.0 - c.dense_dist) >= ?
+        ),
+        best_per_source AS (
+          SELECT *,
+                 ROW_NUMBER() OVER (PARTITION BY source ORDER BY total_score DESC, source, id) AS source_rank
+          FROM scored
+        )
+        SELECT id, source, source_type, file_mtime_at, heading, heading_level, content, line_start, line_end,
+               dense_rank, dense_dist, bm25_rank, recency_rank, total_score, retriever
+        FROM best_per_source
+        WHERE source_rank = 1
+        ORDER BY total_score DESC, source, id
+        LIMIT ?
+      `;
+      const rows = this.db.prepare(sql).all(...params);
+      if (rows.length === 0) return [];
+      const denseActive = hasDense;
+      const bm25Active = hasBm25;
+      const recencyActive = rows.length > 0;
+      const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? recencyWeight : 0)) / (rrfK + 1);
+      return rows.map((r) => {
+        const chunk = this.mapMemoryChunkRow(r);
+        const normalizedScore = maxScore > 0 ? r.total_score / maxScore : 0;
+        const row = {
+          chunk,
+          score: normalizedScore,
+          retriever: r.retriever
+        };
+        if (includeDebug) {
+          const denseScore = r.dense_dist !== null ? Math.max(0, Math.min(1, 1 - r.dense_dist)) : void 0;
+          const bm25Score = r.bm25_rank !== null ? 1 / (rrfK + r.bm25_rank) : void 0;
+          const recencyScore = recencyWeight * (1 / (rrfK + r.recency_rank));
+          const debug = {
+            denseRank: r.dense_rank ?? void 0,
+            denseScore,
+            bm25Rank: r.bm25_rank ?? void 0,
+            bm25Score,
+            recencyRank: r.recency_rank,
+            recencyScore,
+            totalScore: r.total_score
+          };
+          row.debug = debug;
+        }
+        return row;
+      });
+    } catch (err) {
+      console.warn("[memory-store] Hybrid search failed:", err);
+      return [];
+    }
   }
   get vecAvailable() {
     return true;
@@ -15778,14 +15919,14 @@ var MemoryIndexer = class {
     const existingChunkIds = this.store.hashesBySource(source);
     let text;
     try {
-      text = fs6.readFileSync(absolutePath, "utf8");
+      text = await fs6.promises.readFile(absolutePath, "utf8");
     } catch (err) {
       console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
       return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
     }
     let fileMtimeAt;
     try {
-      fileMtimeAt = fs6.statSync(absolutePath).mtimeMs;
+      fileMtimeAt = (await fs6.promises.stat(absolutePath)).mtimeMs;
     } catch (err) {
       console.warn(`[memory-indexer] Cannot stat file ${absolutePath}:`, err);
       fileMtimeAt = Date.now();
@@ -15818,22 +15959,30 @@ var MemoryIndexer = class {
   }
   async indexDirectory(rootDir, opts = {}) {
     const totals = { indexed: 0, deleted: 0, skipped: 0 };
-    if (!fs6.existsSync(rootDir)) return totals;
+    let rootExists;
+    try {
+      await fs6.promises.stat(rootDir);
+      rootExists = true;
+    } catch {
+      rootExists = false;
+    }
+    if (!rootExists) return totals;
     const relativeBase = opts.relativeBase ?? this.config.wikiDir;
     const excludeFiles = new Set(opts.excludeFiles ?? []);
     const files = [];
-    const walk = (dirPath) => {
+    const walk = async (dirPath) => {
       let entries;
       try {
-        entries = fs6.readdirSync(dirPath, { withFileTypes: true });
+        entries = await fs6.promises.readdir(dirPath, { withFileTypes: true });
       } catch (err) {
         console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
         return;
       }
+      entries.sort((a, b) => a.name.localeCompare(b.name));
       for (const entry of entries) {
         const entryPath = path8.join(dirPath, entry.name);
         if (entry.isDirectory()) {
-          walk(entryPath);
+          await walk(entryPath);
           continue;
         }
         if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
@@ -15841,7 +15990,7 @@ var MemoryIndexer = class {
         }
       }
     };
-    walk(rootDir);
+    await walk(rootDir);
     const currentSources = new Set(files.map((file) => path8.relative(relativeBase, file)));
     for (const stale of this.store.indexedSources()) {
       if (!currentSources.has(stale)) {
@@ -15859,77 +16008,34 @@ var MemoryIndexer = class {
   }
   async search(query, topK) {
     const fetchLimit = topK * FETCH_MULTIPLIER;
-    let denseResults = [];
+    let embedding;
     if (this.store.vecAvailable) {
       try {
-        const queryEmbedding = await this.embedder.embed([query]);
-        denseResults = this.store.searchDense(queryEmbedding[0], fetchLimit);
+        const embeddings = await this.embedder.embed([query]);
+        embedding = embeddings[0];
       } catch (err) {
         console.warn("[memory-indexer] Dense search embedding failed:", err);
       }
     }
-    const bm25Results = this.store.searchBm25(query, fetchLimit);
-    const denseIds = new Set(denseResults.map((r) => r.id));
-    const bm25Ids = new Set(bm25Results.map((r) => r.id));
-    const unionIds = /* @__PURE__ */ new Set([...denseIds, ...bm25Ids]);
-    const denseRetrievalScore = new Map(denseResults.map((r) => [r.id, r.score]));
-    const unionArray = [...unionIds];
-    const chunks = this.store.getChunksByIds(unionArray);
-    const chunkMap = new Map(chunks.map((c) => [c.id, c]));
-    const unionIndexOf = new Map(unionArray.map((id, idx) => [id, idx]));
-    const recencyOrdered = [...unionArray].sort((a, b) => {
-      const mtimeA = chunkMap.get(a)?.fileMtimeAt ?? 0;
-      const mtimeB = chunkMap.get(b)?.fileMtimeAt ?? 0;
-      if (mtimeB !== mtimeA) return mtimeB - mtimeA;
-      return (unionIndexOf.get(a) ?? 0) - (unionIndexOf.get(b) ?? 0);
-    });
-    const scoreMap = /* @__PURE__ */ new Map();
-    denseResults.forEach((r, rank) => {
-      scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0, recency: 0 });
-    });
-    bm25Results.forEach((r, rank) => {
-      const entry = scoreMap.get(r.id) ?? { dense: 0, bm25: 0, recency: 0 };
-      entry.bm25 = 1 / (RRF_K + rank + 1);
-      scoreMap.set(r.id, entry);
-    });
-    recencyOrdered.forEach((id, rank) => {
-      const entry = scoreMap.get(id);
-      if (!entry) return;
-      entry.recency = RECENCY_WEIGHT * (1 / (RRF_K + rank + 1));
+    const rows = this.store.searchHybrid({
+      query,
+      embedding,
+      topK,
+      fetchLimit,
+      denseScoreFloor: DENSE_SCORE_FLOOR,
+      recencyWeight: RECENCY_WEIGHT,
+      rrfK: RRF_K
     });
-    const denseActive = denseResults.length > 0;
-    const bm25Active = bm25Results.length > 0;
-    const recencyActive = unionIds.size > 0;
-    const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? RECENCY_WEIGHT : 0)) / (RRF_K + 1);
-    const ranked = [...scoreMap.entries()].map(([id, scores]) => ({
-      id,
-      totalScore: scores.dense + scores.bm25 + scores.recency,
-      hasDense: scores.dense > 0,
-      hasBm25: scores.bm25 > 0,
-      isDenseOnly: denseIds.has(id) && !bm25Ids.has(id)
-    })).filter((candidate) => {
-      if (!candidate.isDenseOnly) return true;
-      const retrieval = denseRetrievalScore.get(candidate.id);
-      if (retrieval === void 0) return true;
-      return retrieval >= DENSE_SCORE_FLOOR;
-    }).sort((a, b) => b.totalScore - a.totalScore);
+    if (rows.length === 0) return [];
     const results = [];
-    const seenSources = /* @__PURE__ */ new Set();
-    for (const r of ranked) {
-      if (results.length >= topK) break;
-      const chunk = chunkMap.get(r.id);
-      if (!chunk) continue;
-      if (seenSources.has(chunk.source)) continue;
-      seenSources.add(chunk.source);
-      const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
-      const retriever = r.hasDense && r.hasBm25 ? "both" : r.hasDense ? "dense" : "bm25";
+    for (const row of rows) {
       let contentSource = "file";
       try {
-        chunk.content = fs6.readFileSync(path8.join(this.config.wikiDir, chunk.source), "utf8");
+        row.chunk.content = await fs6.promises.readFile(path8.join(this.config.wikiDir, row.chunk.source), "utf8");
       } catch {
         contentSource = "fallback";
       }
-      results.push({ chunk, score: normalizedScore, retriever, contentSource });
+      results.push({ chunk: row.chunk, score: row.score, retriever: row.retriever, contentSource });
     }
     return results;
   }
@@ -15938,13 +16044,13 @@ var MemoryIndexer = class {
     const rawDir = path8.join(this.config.wikiDir, "raw");
     const savePath = path8.join(rawDir, `conv_save_${datePart}.md`);
     const lockPath = `${savePath}.lock`;
-    fs6.mkdirSync(rawDir, { recursive: true });
+    await fs6.promises.mkdir(rawDir, { recursive: true });
     const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
     if (!acquired) {
       console.warn("[memory-indexer] Could not acquire write lock, writing anyway (fail-open)");
     }
     try {
-      fs6.appendFileSync(savePath, `
+      await fs6.promises.appendFile(savePath, `
 ${content}
 `, "utf8");
     } finally {

package/dist/mcp/memory.js

@@ -51,7 +51,7 @@ function registerMemoryToolHandlers(server, indexer, store, config, workspaceRoo
             const blocks = [];
             for (const row of rows) {
                 try {
-                    const content = fs.readFileSync(path.join(config.wikiDir, row.source), 'utf8');
+                    const content = await fs.promises.readFile(path.join(config.wikiDir, row.source), 'utf8');
                     const displaySource = row.source.replace(/^compiled\//, '');
                     blocks.push(`### ${displaySource}\n${content}`);
                 }