@hanhnd/agent-kit 1.0.29 → 1.0.30

package/dist/memory/__tests__/indexer.test.js

@@ -268,6 +268,66 @@ describe('MemoryIndexer', () => {
             fs.rmSync(testDir, { recursive: true, force: true });
         }
     });
+    test('search does not include dense-only results when BM25 has matches', async () => {
+        const testCfg = makeConfig('/tmp/search-dense-filter');
+        const chunks = [
+            {
+                id: 'preference-1',
+                source: 'compiled/preferences.md',
+                heading: '',
+                headingLevel: 0,
+                content: 'I like fish',
+                lineStart: 1,
+                lineEnd: 1,
+            },
+            {
+                id: 'dense-only-1',
+                source: 'compiled/entities/worktree.md',
+                heading: 'Worktree',
+                headingLevel: 1,
+                content: 'Unrelated worktree lifecycle content',
+                lineStart: 1,
+                lineEnd: 2,
+            },
+        ];
+        const fakeStore = {
+            vecAvailable: true,
+            searchDense: () => [
+                { id: 'dense-only-1', score: 0.99 },
+                { id: 'preference-1', score: 0.98 },
+            ],
+            searchBm25: () => [{ id: 'preference-1', score: 1 }],
+            getChunksByIds: (ids) => chunks.filter((chunk) => ids.includes(chunk.id)),
+        };
+        const testIndexer = new MemoryIndexer(fakeStore, new StubEmbedder(), testCfg);
+        const results = await testIndexer.search('personal likes and preferences of the user', 5);
+        assert.deepEqual(results.map((result) => result.chunk.id), ['preference-1']);
+        assert.equal(results[0].retriever, 'both');
+    });
+    test('search still returns dense-only results when BM25 has no matches', async () => {
+        const testCfg = makeConfig('/tmp/search-dense-fallback');
+        const chunks = [
+            {
+                id: 'semantic-1',
+                source: 'compiled/preferences.md',
+                heading: '',
+                headingLevel: 0,
+                content: 'I like fish',
+                lineStart: 1,
+                lineEnd: 1,
+            },
+        ];
+        const fakeStore = {
+            vecAvailable: true,
+            searchDense: () => [{ id: 'semantic-1', score: 0.99 }],
+            searchBm25: () => [],
+            getChunksByIds: (ids) => chunks.filter((chunk) => ids.includes(chunk.id)),
+        };
+        const testIndexer = new MemoryIndexer(fakeStore, new StubEmbedder(), testCfg);
+        const results = await testIndexer.search('favorite dish', 5);
+        assert.deepEqual(results.map((result) => result.chunk.id), ['semantic-1']);
+        assert.equal(results[0].retriever, 'dense');
+    });
     test('search falls back to stored chunk content when source file is missing', async () => {
         const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'search-fallback-'));
         const testCfg = makeConfig(path.join(testDir, 'wiki'));

package/dist/memory/__tests__/store.test.js

@@ -72,6 +72,25 @@ describe('MemoryStore', () => {
         assert.ok(results.length > 0, 'Expected at least one BM25 result');
         assert.ok(results.some((r) => r.id === 'bm25-search-0001'), `Expected chunk id in results, got: ${results.map((r) => r.id).join(', ')}`);
     });
+    test('searchBm25 ignores filler words and matches preference source/content terms', () => {
+        const preference = makeChunk({
+            id: 'preference-search-0001',
+            source: 'compiled/preferences.md',
+            heading: '',
+            headingLevel: 0,
+            content: 'I like fish',
+        });
+        const unrelated = makeChunk({
+            id: 'preference-search-0002',
+            source: 'compiled/entities/worktree.md',
+            heading: 'Open Questions',
+            content: 'How should git manage worktree lifecycle decisions?',
+        });
+        store.upsert([preference, unrelated], [new Float32Array(384), new Float32Array(384)]);
+        const results = store.searchBm25('personal likes and preferences of the user', 5);
+        assert.ok(results.length > 0, 'Expected preference query to return results');
+        assert.equal(results[0].id, 'preference-search-0001');
+    });
     test('searchBm25 returns empty array for empty query', () => {
         const results = store.searchBm25('   ', 5);
         assert.deepEqual(results, []);

package/dist/memory/indexer.js

@@ -131,6 +131,10 @@ export class MemoryIndexer {
             }
         }
         const bm25Results = this.store.searchBm25(query, fetchLimit);
+        const bm25Ids = new Set(bm25Results.map((result) => result.id));
+        if (bm25Ids.size > 0) {
+            denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
+        }
         // RRF fusion (k=60)
         const k = 60;
         const scoreMap = new Map();

package/dist/memory/store.js

@@ -1,6 +1,7 @@
 import Database from 'better-sqlite3';
 import * as fs from 'fs';
 import * as path from 'path';
+import { eng, removeStopwords } from 'stopword';
 export class StoreError extends Error {
     constructor(message, cause) {
         super(message);
@@ -58,18 +59,20 @@ export class MemoryStore {
     createSchema(db) {
         db.exec(`
       CREATE TABLE IF NOT EXISTS memory_chunks (
-        rowid       INTEGER PRIMARY KEY AUTOINCREMENT,
-        id          TEXT NOT NULL UNIQUE,
-        source      TEXT NOT NULL,
-        heading     TEXT NOT NULL DEFAULT '',
+        rowid         INTEGER PRIMARY KEY AUTOINCREMENT,
+        id            TEXT NOT NULL UNIQUE,
+        source        TEXT NOT NULL,
+        heading       TEXT NOT NULL DEFAULT '',
         heading_level INTEGER NOT NULL DEFAULT 0,
-        content     TEXT NOT NULL,
-        line_start  INTEGER NOT NULL,
-        line_end    INTEGER NOT NULL,
-        indexed_at  INTEGER NOT NULL
+        content       TEXT NOT NULL,
+        line_start    INTEGER NOT NULL,
+        line_end      INTEGER NOT NULL,
+        indexed_at    INTEGER NOT NULL
       );
 
       CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+        source,
+        heading,
         content,
         content='memory_chunks',
         content_rowid='rowid'
@@ -77,13 +80,14 @@ export class MemoryStore {
 
       CREATE TRIGGER IF NOT EXISTS memory_chunks_ai
         AFTER INSERT ON memory_chunks BEGIN
-          INSERT INTO memory_fts(rowid, content) VALUES (new.rowid, new.content);
+          INSERT INTO memory_fts(rowid, source, heading, content)
+            VALUES (new.rowid, new.source, new.heading, new.content);
         END;
 
       CREATE TRIGGER IF NOT EXISTS memory_chunks_ad
         AFTER DELETE ON memory_chunks BEGIN
-          INSERT INTO memory_fts(memory_fts, rowid, content)
-            VALUES ('delete', old.rowid, old.content);
+          INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
+            VALUES ('delete', old.rowid, old.source, old.heading, old.content);
         END;
     `);
         if (this._vecAvailable) {
@@ -206,18 +210,19 @@ export class MemoryStore {
     searchBm25(query, limit) {
         if (!query.trim())
             return [];
-        // FTS5 defaults to AND for multi-word queries; convert to OR so any term is a hit
-        const ftsQuery = query
+        const tokens = query
             .trim()
             .split(/\s+/)
-            .map((t) => t.replace(/['"*^():\-]/g, ' ').trim())
-            .filter(Boolean)
+            .map((t) => t.replace(/[^a-zA-Z0-9_]/g, '').toLowerCase())
+            .filter((t) => t.length > 1);
+        const ftsQuery = removeStopwords(tokens, eng)
+            .map((t) => `${t.replace(/s$/, '')}*`)
             .join(' OR ');
         if (!ftsQuery)
             return [];
         try {
             const rows = this.db
-                .prepare(`SELECT mc.id, rank
+                .prepare(`SELECT mc.id, bm25(memory_fts, 0.2, 2.0, 5.0) AS rank
            FROM memory_fts
            JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
            WHERE memory_fts MATCH ?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hanhnd/agent-kit",
-  "version": "1.0.29",
+  "version": "1.0.30",
   "description": "Agent Kit MCP server for software development agent workflows",
   "type": "module",
   "main": "dist/kit-server.js",
@@ -32,6 +32,7 @@
     "dedent": "^1.5.3",
     "fastembed": "^1.14.0",
     "sqlite-vec": "^0.1.0",
+    "stopword": "^3.1.5",
     "zod": "^3.24.4"
   },
   "devDependencies": {