@hanhnd/agent-kit 1.0.37 → 1.0.39

package/dist/server.js

@@ -25930,8 +25930,8 @@ function getElementAtPath(obj, path15) {
 }
 function promiseAllObject(promisesObj) {
   const keys = Object.keys(promisesObj);
-  const promises = keys.map((key) => promisesObj[key]);
-  return Promise.all(promises).then((results) => {
+  const promises3 = keys.map((key) => promisesObj[key]);
+  return Promise.all(promises3).then((results) => {
     const resolvedObj = {};
     for (let i = 0; i < keys.length; i++) {
       resolvedObj[keys[i]] = results[i];
@@ -35920,6 +35920,9 @@ var DEFAULT_MEMORY_CONFIG = {
 var LOCK_RETRY_MS = 50;
 var LOCK_TIMEOUT_MS = 500;
 var RRF_K = 60;
+var FETCH_MULTIPLIER = 4;
+var RECENCY_WEIGHT = 0.5;
+var DENSE_SCORE_FLOOR = 0.2;
 
 // src/utils/files.ts
 import * as fs from "node:fs";
@@ -37013,14 +37016,14 @@ var MemoryIndexer = class {
     const existingChunkIds = this.store.hashesBySource(source);
     let text;
     try {
-      text = fs6.readFileSync(absolutePath, "utf8");
+      text = await fs6.promises.readFile(absolutePath, "utf8");
     } catch (err) {
       console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
       return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
     }
     let fileMtimeAt;
     try {
-      fileMtimeAt = fs6.statSync(absolutePath).mtimeMs;
+      fileMtimeAt = (await fs6.promises.stat(absolutePath)).mtimeMs;
     } catch (err) {
       console.warn(`[memory-indexer] Cannot stat file ${absolutePath}:`, err);
       fileMtimeAt = Date.now();
@@ -37053,22 +37056,30 @@ var MemoryIndexer = class {
   }
   async indexDirectory(rootDir, opts = {}) {
     const totals = { indexed: 0, deleted: 0, skipped: 0 };
-    if (!fs6.existsSync(rootDir)) return totals;
+    let rootExists;
+    try {
+      await fs6.promises.stat(rootDir);
+      rootExists = true;
+    } catch {
+      rootExists = false;
+    }
+    if (!rootExists) return totals;
     const relativeBase = opts.relativeBase ?? this.config.wikiDir;
     const excludeFiles = new Set(opts.excludeFiles ?? []);
     const files = [];
-    const walk = (dirPath) => {
+    const walk = async (dirPath) => {
       let entries;
       try {
-        entries = fs6.readdirSync(dirPath, { withFileTypes: true });
+        entries = await fs6.promises.readdir(dirPath, { withFileTypes: true });
       } catch (err) {
         console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
         return;
       }
+      entries.sort((a, b) => a.name.localeCompare(b.name));
       for (const entry of entries) {
         const entryPath = path8.join(dirPath, entry.name);
         if (entry.isDirectory()) {
-          walk(entryPath);
+          await walk(entryPath);
           continue;
         }
         if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
@@ -37076,7 +37087,7 @@ var MemoryIndexer = class {
         }
       }
     };
-    walk(rootDir);
+    await walk(rootDir);
     const currentSources = new Set(files.map((file) => path8.relative(relativeBase, file)));
     for (const stale of this.store.indexedSources()) {
       if (!currentSources.has(stale)) {
@@ -37093,58 +37104,35 @@ var MemoryIndexer = class {
     return totals;
   }
   async search(query, topK) {
-    const fetchLimit = topK * 2;
-    let denseResults = [];
+    const fetchLimit = topK * FETCH_MULTIPLIER;
+    let embedding;
     if (this.store.vecAvailable) {
       try {
-        const queryEmbedding = await this.embedder.embed([query]);
-        denseResults = this.store.searchDense(queryEmbedding[0], fetchLimit);
+        const embeddings = await this.embedder.embed([query]);
+        embedding = embeddings[0];
       } catch (err) {
         console.warn("[memory-indexer] Dense search embedding failed:", err);
       }
     }
-    const bm25Results = this.store.searchBm25(query, fetchLimit);
-    const bm25Ids = new Set(bm25Results.map((result) => result.id));
-    if (bm25Ids.size > 0) {
-      denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
-    }
-    const scoreMap = /* @__PURE__ */ new Map();
-    denseResults.forEach((r, rank) => {
-      scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0 });
-    });
-    bm25Results.forEach((r, rank) => {
-      const entry = scoreMap.get(r.id) ?? { dense: 0, bm25: 0 };
-      entry.bm25 = 1 / (RRF_K + rank + 1);
-      scoreMap.set(r.id, entry);
+    const rows = this.store.searchHybrid({
+      query,
+      embedding,
+      topK,
+      fetchLimit,
+      denseScoreFloor: DENSE_SCORE_FLOOR,
+      recencyWeight: RECENCY_WEIGHT,
+      rrfK: RRF_K
     });
-    const hasDense = this.store.vecAvailable && denseResults.length > 0;
-    const numRetrievers = hasDense ? 2 : 1;
-    const maxScore = numRetrievers / (RRF_K + 1);
-    const ranked = [...scoreMap.entries()].map(([id, scores]) => ({
-      id,
-      totalScore: scores.dense + scores.bm25,
-      hasDense: scores.dense > 0,
-      hasBm25: scores.bm25 > 0
-    })).sort((a, b) => b.totalScore - a.totalScore);
-    const chunks = this.store.getChunksByIds(ranked.map((r) => r.id));
-    const chunkMap = new Map(chunks.map((c) => [c.id, c]));
+    if (rows.length === 0) return [];
     const results = [];
-    const seenSources = /* @__PURE__ */ new Set();
-    for (const r of ranked) {
-      if (results.length >= topK) break;
-      const chunk = chunkMap.get(r.id);
-      if (!chunk) continue;
-      if (seenSources.has(chunk.source)) continue;
-      seenSources.add(chunk.source);
-      const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
-      const retriever = r.hasDense && r.hasBm25 ? "both" : r.hasDense ? "dense" : "bm25";
+    for (const row of rows) {
       let contentSource = "file";
       try {
-        chunk.content = fs6.readFileSync(path8.join(this.config.wikiDir, chunk.source), "utf8");
+        row.chunk.content = await fs6.promises.readFile(path8.join(this.config.wikiDir, row.chunk.source), "utf8");
       } catch {
         contentSource = "fallback";
       }
-      results.push({ chunk, score: normalizedScore, retriever, contentSource });
+      results.push({ chunk: row.chunk, score: row.score, retriever: row.retriever, contentSource });
     }
     return results;
   }
@@ -37153,13 +37141,13 @@ var MemoryIndexer = class {
     const rawDir = path8.join(this.config.wikiDir, "raw");
     const savePath = path8.join(rawDir, `conv_save_${datePart}.md`);
     const lockPath = `${savePath}.lock`;
-    fs6.mkdirSync(rawDir, { recursive: true });
+    await fs6.promises.mkdir(rawDir, { recursive: true });
     const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
     if (!acquired) {
       console.warn("[memory-indexer] Could not acquire write lock, writing anyway (fail-open)");
     }
     try {
-      fs6.appendFileSync(savePath, `
+      await fs6.promises.appendFile(savePath, `
 ${content}
 `, "utf8");
     } finally {
@@ -37321,6 +37309,7 @@ var MemoryStore = class {
     return 0;
   }
   upsert(chunks, embeddings) {
+    const deleteChunk = this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`);
     const insertChunk = this.db.prepare(`
       INSERT OR REPLACE INTO memory_chunks
         (id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
@@ -37331,7 +37320,7 @@ var MemoryStore = class {
         const chunk = chunks[i];
         try {
           const vectorBinding = this.toVectorBinding(embeddings[i]);
-          this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
+          deleteChunk.run(chunk.id);
           insertChunk.run(
             chunk.id,
             chunk.source,
@@ -37360,15 +37349,10 @@ var MemoryStore = class {
     if (!embedding) throw new Error("missing embedding");
     if (!(embedding instanceof Float32Array)) throw new Error("missing embedding");
     if (embedding.length !== this.config.vectorDimension) throw new Error("embedding dimension mismatch");
-    const values = Array.from(embedding, (value) => {
-      if (!Number.isFinite(value)) throw new Error("embedding contains non-finite value");
-      return value;
-    });
-    return JSON.stringify(values);
-  }
-  normalizeVectorDistance(distance) {
-    if (distance <= 0) return 1;
-    return Math.max(0, Math.min(1, 1 - distance));
+    for (let i = 0; i < embedding.length; i++) {
+      if (!Number.isFinite(embedding[i])) throw new Error("embedding contains non-finite value");
+    }
+    return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
   }
   hashesBySource(source) {
     const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
@@ -37406,61 +37390,12 @@ var MemoryStore = class {
     });
     deleteAll();
   }
-  searchDense(embedding, limit) {
-    try {
-      const queryVector = this.toVectorBinding(embedding);
-      const rows = this.db.prepare(
-        `SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
-           FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
-           JOIN memory_chunks mc ON mc.rowid = vector_matches.id
-           ORDER BY distance
-           LIMIT ?`
-      ).all(queryVector, queryVector, limit, limit);
-      return rows.map((r) => ({
-        id: r.id,
-        score: this.normalizeVectorDistance(r.distance)
-      }));
-    } catch (err) {
-      console.warn("[memory-store] Dense search failed:", err);
-      return [];
-    }
-  }
-  searchBm25(query, limit) {
-    if (!query.trim()) return [];
+  buildFtsQuery(query) {
     const tokens = query.trim().split(/\s+/).map((t) => t.replace(/[^a-zA-Z0-9_]/g, "").toLowerCase()).filter((t) => t.length > 1);
-    const ftsQuery = (0, import_stopword.removeStopwords)(tokens, import_stopword.eng).map((t) => `${t.replace(/s$/, "")}*`).join(" OR ");
-    if (!ftsQuery) return [];
-    try {
-      const rows = this.db.prepare(
-        `SELECT mc.id, bm25(memory_fts, 0.2, 2.0, 5.0) AS rank
-           FROM memory_fts
-           JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
-           WHERE memory_fts MATCH ?
-           ORDER BY rank
-           LIMIT ?`
-      ).all(ftsQuery, limit);
-      if (rows.length === 0) return [];
-      const ranks = rows.map((r) => r.rank);
-      const minRank = Math.min(...ranks);
-      const maxRank = Math.max(...ranks);
-      const range = maxRank - minRank;
-      return rows.map((r) => ({
-        id: r.id,
-        score: range > 0 ? (maxRank - r.rank) / range : 1
-      }));
-    } catch (err) {
-      console.warn("[memory-store] BM25 search failed:", err);
-      return [];
-    }
+    return (0, import_stopword.removeStopwords)(tokens, import_stopword.eng).map((t) => `${t.replace(/s$/, "")}*`).join(" OR ");
   }
-  getChunksByIds(ids) {
-    if (ids.length === 0) return [];
-    const placeholders = ids.map(() => "?").join(", ");
-    const rows = this.db.prepare(
-      `SELECT id, source, source_type, heading, heading_level, content, line_start, line_end, file_mtime_at
-         FROM memory_chunks WHERE id IN (${placeholders})`
-    ).all(...ids);
-    return rows.map((r) => ({
+  mapMemoryChunkRow(r) {
+    return {
       id: r.id,
       source: r.source,
       sourceType: r.source_type,
@@ -37470,7 +37405,201 @@ var MemoryStore = class {
       lineStart: r.line_start,
       lineEnd: r.line_end,
       fileMtimeAt: r.file_mtime_at
-    }));
+    };
+  }
+  searchHybrid(options) {
+    const {
+      query,
+      embedding,
+      topK,
+      fetchLimit,
+      denseScoreFloor,
+      recencyWeight,
+      rrfK,
+      sourceType,
+      sinceMtimeAt,
+      includeDebug
+    } = options;
+    const ftsQuery = this.buildFtsQuery(query);
+    const hasDense = embedding !== void 0;
+    const hasBm25 = ftsQuery.length > 0;
+    if (!hasDense && !hasBm25) return [];
+    const hasFilters = sourceType !== void 0 || sinceMtimeAt !== void 0;
+    try {
+      const params = [];
+      let denseCte = "";
+      let denseRankedCte = "";
+      if (hasDense) {
+        const queryBuf = this.toVectorBinding(embedding);
+        if (hasFilters) {
+          const filterClauses = [];
+          if (sourceType) {
+            filterClauses.push("mc.source_type = ?");
+            params.push(sourceType);
+          }
+          if (sinceMtimeAt !== void 0) {
+            filterClauses.push("mc.file_mtime_at >= ?");
+            params.push(sinceMtimeAt);
+          }
+          const whereClause = filterClauses.length > 0 ? `WHERE ${filterClauses.join(" AND ")}` : "";
+          params.push(queryBuf);
+          params.push(fetchLimit);
+          denseCte = `dense_data AS (
+            SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                   mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                   vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
+            FROM memory_chunks mc
+            ${whereClause}
+            ORDER BY dense_dist
+            LIMIT ?
+          )`;
+        } else {
+          params.push(queryBuf);
+          params.push(queryBuf);
+          params.push(fetchLimit);
+          denseCte = `dense_data AS (
+            SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                   mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                   vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
+            FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vtk
+            JOIN memory_chunks mc ON mc.rowid = vtk.id
+          )`;
+        }
+        denseRankedCte = `dense_ranked AS (
+          SELECT *, ROW_NUMBER() OVER (ORDER BY dense_dist, id) AS dense_rank
+          FROM dense_data
+        )`;
+      }
+      let bm25Cte = "";
+      let bm25RankedCte = "";
+      if (hasBm25) {
+        const bm25FilterClauses = [];
+        if (sourceType) bm25FilterClauses.push("mc.source_type = ?");
+        if (sinceMtimeAt !== void 0) bm25FilterClauses.push("mc.file_mtime_at >= ?");
+        const bm25WhereExtra = bm25FilterClauses.length > 0 ? ` AND ${bm25FilterClauses.join(" AND ")}` : "";
+        params.push(ftsQuery);
+        if (sourceType) params.push(sourceType);
+        if (sinceMtimeAt !== void 0) params.push(sinceMtimeAt);
+        params.push(fetchLimit);
+        bm25Cte = `bm25_data AS (
+          SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
+                 mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
+                 bm25(memory_fts, 0.2, 2.0, 5.0) AS bm25_neg
+          FROM memory_fts
+          JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
+          WHERE memory_fts MATCH ?${bm25WhereExtra}
+          LIMIT ?
+        )`;
+        bm25RankedCte = `bm25_ranked AS (
+          SELECT *, ROW_NUMBER() OVER (ORDER BY bm25_neg, id) AS bm25_rank
+          FROM bm25_data
+        )`;
+      }
+      let candidatesCte = "";
+      if (hasDense && hasBm25) {
+        candidatesCte = `all_candidates AS (
+          SELECT d.id, d.source, d.source_type, d.file_mtime_at,
+                 d.heading, d.heading_level, d.content, d.line_start, d.line_end,
+                 CAST(d.dense_rank AS INTEGER) AS dense_rank, d.dense_dist,
+                 b.bm25_rank,
+                 CASE WHEN b.id IS NOT NULL THEN 'both' ELSE 'dense' END AS retriever
+          FROM dense_ranked d
+          LEFT JOIN bm25_ranked b ON b.id = d.id
+          UNION ALL
+          SELECT b.id, b.source, b.source_type, b.file_mtime_at,
+                 b.heading, b.heading_level, b.content, b.line_start, b.line_end,
+                 NULL AS dense_rank, NULL AS dense_dist,
+                 CAST(b.bm25_rank AS INTEGER) AS bm25_rank,
+                 'bm25' AS retriever
+          FROM bm25_ranked b
+          LEFT JOIN dense_ranked d ON d.id = b.id
+          WHERE d.id IS NULL
+        )`;
+      } else if (hasDense) {
+        candidatesCte = `all_candidates AS (
+          SELECT id, source, source_type, file_mtime_at,
+                 heading, heading_level, content, line_start, line_end,
+                 CAST(dense_rank AS INTEGER) AS dense_rank, dense_dist,
+                 NULL AS bm25_rank, 'dense' AS retriever
+          FROM dense_ranked
+        )`;
+      } else {
+        candidatesCte = `all_candidates AS (
+          SELECT id, source, source_type, file_mtime_at,
+                 heading, heading_level, content, line_start, line_end,
+                 NULL AS dense_rank, NULL AS dense_dist,
+                 CAST(bm25_rank AS INTEGER) AS bm25_rank, 'bm25' AS retriever
+          FROM bm25_ranked
+        )`;
+      }
+      params.push(rrfK, rrfK, recencyWeight, rrfK, denseScoreFloor, topK);
+      const sql = `
+        WITH
+          ${[denseCte, denseRankedCte, bm25Cte, bm25RankedCte, candidatesCte].filter(Boolean).join(",\n")}
+        ,
+        recency_ranked AS (
+          SELECT id, ROW_NUMBER() OVER (ORDER BY file_mtime_at DESC, source, id) AS recency_rank
+          FROM all_candidates
+        ),
+        scored AS (
+          SELECT c.id, c.source, c.source_type, c.file_mtime_at,
+                 c.heading, c.heading_level, c.content, c.line_start, c.line_end,
+                 c.dense_rank, c.dense_dist, c.bm25_rank, r.recency_rank,
+                 c.retriever,
+                 COALESCE(1.0 / (? + c.dense_rank), 0.0) +
+                 COALESCE(1.0 / (? + c.bm25_rank), 0.0) +
+                 ? * (1.0 / (? + r.recency_rank)) AS total_score
+          FROM all_candidates c
+          JOIN recency_ranked r ON r.id = c.id
+          WHERE c.retriever != 'dense' OR (1.0 - c.dense_dist) >= ?
+        ),
+        best_per_source AS (
+          SELECT *,
+                 ROW_NUMBER() OVER (PARTITION BY source ORDER BY total_score DESC, source, id) AS source_rank
+          FROM scored
+        )
+        SELECT id, source, source_type, file_mtime_at, heading, heading_level, content, line_start, line_end,
+               dense_rank, dense_dist, bm25_rank, recency_rank, total_score, retriever
+        FROM best_per_source
+        WHERE source_rank = 1
+        ORDER BY total_score DESC, source, id
+        LIMIT ?
+      `;
+      const rows = this.db.prepare(sql).all(...params);
+      if (rows.length === 0) return [];
+      const denseActive = hasDense;
+      const bm25Active = hasBm25;
+      const recencyActive = rows.length > 0;
+      const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? recencyWeight : 0)) / (rrfK + 1);
+      return rows.map((r) => {
+        const chunk = this.mapMemoryChunkRow(r);
+        const normalizedScore = maxScore > 0 ? r.total_score / maxScore : 0;
+        const row = {
+          chunk,
+          score: normalizedScore,
+          retriever: r.retriever
+        };
+        if (includeDebug) {
+          const denseScore = r.dense_dist !== null ? Math.max(0, Math.min(1, 1 - r.dense_dist)) : void 0;
+          const bm25Score = r.bm25_rank !== null ? 1 / (rrfK + r.bm25_rank) : void 0;
+          const recencyScore = recencyWeight * (1 / (rrfK + r.recency_rank));
+          const debug = {
+            denseRank: r.dense_rank ?? void 0,
+            denseScore,
+            bm25Rank: r.bm25_rank ?? void 0,
+            bm25Score,
+            recencyRank: r.recency_rank,
+            recencyScore,
+            totalScore: r.total_score
+          };
+          row.debug = debug;
+        }
+        return row;
+      });
+    } catch (err) {
+      console.warn("[memory-store] Hybrid search failed:", err);
+      return [];
+    }
   }
   get vecAvailable() {
     return true;
@@ -37955,7 +38084,7 @@ ${content}`;
         const blocks = [];
         for (const row of rows) {
           try {
-            const content = fs11.readFileSync(path14.join(config2.wikiDir, row.source), "utf8");
+            const content = await fs11.promises.readFile(path14.join(config2.wikiDir, row.source), "utf8");
             const displaySource = row.source.replace(/^compiled\//, "");
             blocks.push(`### ${displaySource}
 ${content}`);

package/dist/services/digest/digest-processor.test.js

@@ -72,10 +72,8 @@ describe('digestConversationFile', () => {
             });
             assert.equal(result.indexed, true);
             assert.equal(result.markdown, markdownPath);
-            const denseResults = store.searchDense(new Float32Array(384).fill(0.05), 5);
             const indexedSource = path.relative(config.wikiDir, markdownPath);
-            const indexedChunks = store.getChunksByIds(denseResults.map((denseResult) => denseResult.id));
-            assert.ok(indexedChunks.some((chunk) => chunk.source === indexedSource), `Expected dense search to return a chunk for ${indexedSource}`);
+            assert.ok(store.hashesBySource(indexedSource).size > 0, `Expected source ${indexedSource} to be indexed in store`);
         }
         finally {
             store.close();

package/dist/services/memory/constants.d.ts

@@ -4,3 +4,6 @@ export declare const DEFAULT_MEMORY_CONFIG: MemoryConfig;
 export declare const LOCK_RETRY_MS = 50;
 export declare const LOCK_TIMEOUT_MS = 500;
 export declare const RRF_K = 60;
+export declare const FETCH_MULTIPLIER = 4;
+export declare const RECENCY_WEIGHT = 0.5;
+export declare const DENSE_SCORE_FLOOR = 0.2;

package/dist/services/memory/constants.js

@@ -16,3 +16,9 @@ export const DEFAULT_MEMORY_CONFIG = {
 export const LOCK_RETRY_MS = 50;
 export const LOCK_TIMEOUT_MS = 500;
 export const RRF_K = 60;
+// Tunable starting default: fetchLimit = topK * FETCH_MULTIPLIER (was hardcoded topK * 2)
+export const FETCH_MULTIPLIER = 4;
+// Tunable starting default: weight on the recency RRF channel; < 1 so recency cannot outrank a dual-channel durable hit
+export const RECENCY_WEIGHT = 0.5;
+// Tunable starting default: min dense retrieval score [0,1] for a dense-only candidate to survive (permissive start)
+export const DENSE_SCORE_FLOOR = 0.2;