@hanhnd/agent-kit 1.0.37 → 1.0.39

package/dist/services/memory/indexer.js

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { chunkMarkdown } from './chunker.js';
-import { LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RRF_K } from './constants.js';
+import { DENSE_SCORE_FLOOR, FETCH_MULTIPLIER, LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RECENCY_WEIGHT, RRF_K, } from './constants.js';
 import { acquireLock, releaseLock } from '../../utils/files.js';
 export function deriveSourceType(source) {
     if (source.startsWith('compiled/provisional/conversation-digests/'))
@@ -32,7 +32,7 @@ export class MemoryIndexer {
         const existingChunkIds = this.store.hashesBySource(source);
         let text;
         try {
-            text = fs.readFileSync(absolutePath, 'utf8');
+            text = await fs.promises.readFile(absolutePath, 'utf8');
         }
         catch (err) {
             console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
@@ -40,7 +40,7 @@ export class MemoryIndexer {
         }
         let fileMtimeAt;
         try {
-            fileMtimeAt = fs.statSync(absolutePath).mtimeMs;
+            fileMtimeAt = (await fs.promises.stat(absolutePath)).mtimeMs;
         }
         catch (err) {
             console.warn(`[memory-indexer] Cannot stat file ${absolutePath}:`, err);
@@ -75,24 +75,34 @@ export class MemoryIndexer {
     }
     async indexDirectory(rootDir, opts = {}) {
         const totals = { indexed: 0, deleted: 0, skipped: 0 };
-        if (!fs.existsSync(rootDir))
+        let rootExists;
+        try {
+            await fs.promises.stat(rootDir);
+            rootExists = true;
+        }
+        catch {
+            rootExists = false;
+        }
+        if (!rootExists)
             return totals;
         const relativeBase = opts.relativeBase ?? this.config.wikiDir;
         const excludeFiles = new Set(opts.excludeFiles ?? []);
         const files = [];
-        const walk = (dirPath) => {
+        const walk = async (dirPath) => {
             let entries;
             try {
-                entries = fs.readdirSync(dirPath, { withFileTypes: true });
+                entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
             }
             catch (err) {
                 console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
                 return;
             }
+            // Sort for deterministic traversal order
+            entries.sort((a, b) => a.name.localeCompare(b.name));
             for (const entry of entries) {
                 const entryPath = path.join(dirPath, entry.name);
                 if (entry.isDirectory()) {
-                    walk(entryPath);
+                    await walk(entryPath);
                     continue;
                 }
                 if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
@@ -100,7 +110,7 @@ export class MemoryIndexer {
                 }
             }
         };
-        walk(rootDir);
+        await walk(rootDir);
         const currentSources = new Set(files.map((file) => path.relative(relativeBase, file)));
         // Remove stale sources (deleted files or pre-migration daily-file sources)
         for (const stale of this.store.indexedSources()) {
@@ -118,65 +128,38 @@ export class MemoryIndexer {
         return totals;
     }
     async search(query, topK) {
-        const fetchLimit = topK * 2;
-        let denseResults = [];
+        const fetchLimit = topK * FETCH_MULTIPLIER;
+        let embedding;
         if (this.store.vecAvailable) {
             try {
-                const queryEmbedding = await this.embedder.embed([query]);
-                denseResults = this.store.searchDense(queryEmbedding[0], fetchLimit);
+                const embeddings = await this.embedder.embed([query]);
+                embedding = embeddings[0];
             }
             catch (err) {
                 console.warn('[memory-indexer] Dense search embedding failed:', err);
             }
         }
-        const bm25Results = this.store.searchBm25(query, fetchLimit);
-        const bm25Ids = new Set(bm25Results.map((result) => result.id));
-        if (bm25Ids.size > 0) {
-            denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
-        }
-        const scoreMap = new Map();
-        denseResults.forEach((r, rank) => {
-            scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0 });
-        });
-        bm25Results.forEach((r, rank) => {
-            const entry = scoreMap.get(r.id) ?? { dense: 0, bm25: 0 };
-            entry.bm25 = 1 / (RRF_K + rank + 1);
-            scoreMap.set(r.id, entry);
+        const rows = this.store.searchHybrid({
+            query,
+            embedding,
+            topK,
+            fetchLimit,
+            denseScoreFloor: DENSE_SCORE_FLOOR,
+            recencyWeight: RECENCY_WEIGHT,
+            rrfK: RRF_K,
         });
-        const hasDense = this.store.vecAvailable && denseResults.length > 0;
-        const numRetrievers = hasDense ? 2 : 1;
-        const maxScore = numRetrievers / (RRF_K + 1);
-        const ranked = [...scoreMap.entries()]
-            .map(([id, scores]) => ({
-            id,
-            totalScore: scores.dense + scores.bm25,
-            hasDense: scores.dense > 0,
-            hasBm25: scores.bm25 > 0,
-        }))
-            .sort((a, b) => b.totalScore - a.totalScore);
-        const chunks = this.store.getChunksByIds(ranked.map((r) => r.id));
-        const chunkMap = new Map(chunks.map((c) => [c.id, c]));
+        if (rows.length === 0)
+            return [];
         const results = [];
-        const seenSources = new Set();
-        for (const r of ranked) {
-            if (results.length >= topK)
-                break;
-            const chunk = chunkMap.get(r.id);
-            if (!chunk)
-                continue;
-            if (seenSources.has(chunk.source))
-                continue;
-            seenSources.add(chunk.source);
-            const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
-            const retriever = r.hasDense && r.hasBm25 ? 'both' : r.hasDense ? 'dense' : 'bm25';
+        for (const row of rows) {
             let contentSource = 'file';
             try {
-                chunk.content = fs.readFileSync(path.join(this.config.wikiDir, chunk.source), 'utf8');
+                row.chunk.content = await fs.promises.readFile(path.join(this.config.wikiDir, row.chunk.source), 'utf8');
             }
             catch {
                 contentSource = 'fallback';
             }
-            results.push({ chunk, score: normalizedScore, retriever, contentSource });
+            results.push({ chunk: row.chunk, score: row.score, retriever: row.retriever, contentSource });
         }
         return results;
     }
@@ -185,13 +168,13 @@ export class MemoryIndexer {
         const rawDir = path.join(this.config.wikiDir, 'raw');
         const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
         const lockPath = `${savePath}.lock`;
-        fs.mkdirSync(rawDir, { recursive: true });
+        await fs.promises.mkdir(rawDir, { recursive: true });
         const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
         if (!acquired) {
             console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
         }
         try {
-            fs.appendFileSync(savePath, `\n${content}\n`, 'utf8');
+            await fs.promises.appendFile(savePath, `\n${content}\n`, 'utf8');
         }
         finally {
             if (acquired)