@hanhnd/agent-kit 1.0.38 → 1.0.40

package/dist/services/digest/digest-processor.test.js

@@ -72,10 +72,8 @@ describe('digestConversationFile', () => {
             });
             assert.equal(result.indexed, true);
             assert.equal(result.markdown, markdownPath);
-            const denseResults = store.searchDense(new Float32Array(384).fill(0.05), 5);
             const indexedSource = path.relative(config.wikiDir, markdownPath);
-            const indexedChunks = store.getChunksByIds(denseResults.map((denseResult) => denseResult.id));
-            assert.ok(indexedChunks.some((chunk) => chunk.source === indexedSource), `Expected dense search to return a chunk for ${indexedSource}`);
+            assert.ok(store.hashesBySource(indexedSource).size > 0, `Expected source ${indexedSource} to be indexed in store`);
         }
         finally {
             store.close();

package/dist/services/memory/index.d.ts

@@ -1,5 +1,7 @@
 export { MemoryStore, StoreError } from './store.js';
 export { Embedder, EmbedderError } from './embedder.js';
 export { MemoryIndexer } from './indexer.js';
-export type { MemoryChunk, SearchResult, MemoryConfig, IndexStats, SourceType, RecentSource } from './types.js';
+export { createMemorySubsystem } from './subsystem.js';
+export type { MemorySubsystem, MemorySubsystemOverrides } from './subsystem.js';
+export type { MemoryChunk, SearchResult, MemoryConfig, IndexStats, SourceType, RecentSource, MemoryLifecycleState, MemoryLifecycleStatus, IndexDirectoryOptions, PreparedIndexMutation, } from './types.js';
 export { DEFAULT_MEMORY_CONFIG } from './constants.js';

package/dist/services/memory/index.js

@@ -1,4 +1,5 @@
 export { MemoryStore, StoreError } from './store.js';
 export { Embedder, EmbedderError } from './embedder.js';
 export { MemoryIndexer } from './indexer.js';
+export { createMemorySubsystem } from './subsystem.js';
 export { DEFAULT_MEMORY_CONFIG } from './constants.js';

package/dist/services/memory/indexer.d.ts

@@ -1,5 +1,5 @@
 import type { MemoryStore } from './store.js';
-import type { IndexStats, MemoryConfig, SearchResult, SourceType } from './types.js';
+import type { IndexDirectoryOptions, IndexStats, MemoryConfig, SearchResult, SourceType } from './types.js';
 interface TextEmbedder {
     embed(texts: string[]): Promise<Float32Array[]>;
     initialize(): Promise<void>;
@@ -12,11 +12,10 @@ export declare class MemoryIndexer {
     private _ready;
     constructor(store: MemoryStore, embedder: TextEmbedder, config: MemoryConfig);
     indexFile(absolutePath: string): Promise<IndexStats>;
-    private indexFileRelativeTo;
-    indexDirectory(rootDir: string, opts?: {
-        relativeBase?: string;
-        excludeFiles?: string[];
-    }): Promise<IndexStats>;
+    private makeSkippedMutation;
+    private prepareIndexFileRelativeTo;
+    private commitPreparedMutation;
+    indexDirectory(rootDir: string, opts?: IndexDirectoryOptions): Promise<IndexStats>;
     search(query: string, topK: number): Promise<SearchResult[]>;
     save(content: string): Promise<IndexStats>;
     startupIndex(): Promise<void>;

package/dist/services/memory/indexer.js

@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { chunkMarkdown } from './chunker.js';
 import { DENSE_SCORE_FLOOR, FETCH_MULTIPLIER, LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RECENCY_WEIGHT, RRF_K, } from './constants.js';
+import { mapLimit } from '../../utils/async.js';
 import { acquireLock, releaseLock } from '../../utils/files.js';
 export function deriveSourceType(source) {
     if (source.startsWith('compiled/provisional/conversation-digests/'))
@@ -25,22 +26,32 @@ export class MemoryIndexer {
         this.config = config;
     }
     async indexFile(absolutePath) {
-        return this.indexFileRelativeTo(absolutePath, this.config.wikiDir);
+        const mutation = await this.prepareIndexFileRelativeTo(absolutePath, this.config.wikiDir);
+        return this.commitPreparedMutation(mutation);
     }
-    async indexFileRelativeTo(absolutePath, sourceRoot) {
+    makeSkippedMutation(source, skipped) {
+        return {
+            source,
+            chunksToUpsert: [],
+            embeddings: [],
+            idsToDelete: [],
+            stats: { indexed: 0, deleted: 0, skipped },
+        };
+    }
+    async prepareIndexFileRelativeTo(absolutePath, sourceRoot) {
         const source = path.relative(sourceRoot, absolutePath);
         const existingChunkIds = this.store.hashesBySource(source);
         let text;
         try {
-            text = fs.readFileSync(absolutePath, 'utf8');
+            text = await fs.promises.readFile(absolutePath, 'utf8');
         }
         catch (err) {
             console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
-            return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
+            return this.makeSkippedMutation(source, existingChunkIds.size);
         }
         let fileMtimeAt;
         try {
-            fileMtimeAt = fs.statSync(absolutePath).mtimeMs;
+            fileMtimeAt = (await fs.promises.stat(absolutePath)).mtimeMs;
         }
         catch (err) {
             console.warn(`[memory-indexer] Cannot stat file ${absolutePath}:`, err);
@@ -58,41 +69,60 @@ export class MemoryIndexer {
             }
             catch (err) {
                 console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
-                return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
+                return this.makeSkippedMutation(source, existingChunkIds.size);
             }
         }
-        if (toIndex.length > 0) {
-            this.store.upsert(toIndex, embeddings);
-        }
-        if (toDelete.length > 0) {
-            this.store.deleteByIds(toDelete);
-        }
         return {
-            indexed: toIndex.length,
-            deleted: toDelete.length,
-            skipped: allChunks.length - toIndex.length,
+            source,
+            chunksToUpsert: toIndex,
+            embeddings,
+            idsToDelete: toDelete,
+            stats: {
+                indexed: toIndex.length,
+                deleted: toDelete.length,
+                skipped: allChunks.length - toIndex.length,
+            },
         };
     }
+    commitPreparedMutation(mutation) {
+        if (mutation.chunksToUpsert.length > 0) {
+            this.store.upsert(mutation.chunksToUpsert, mutation.embeddings);
+        }
+        if (mutation.idsToDelete.length > 0) {
+            this.store.deleteByIds(mutation.idsToDelete);
+        }
+        return mutation.stats;
+    }
     async indexDirectory(rootDir, opts = {}) {
         const totals = { indexed: 0, deleted: 0, skipped: 0 };
-        if (!fs.existsSync(rootDir))
+        let rootExists;
+        try {
+            await fs.promises.stat(rootDir);
+            rootExists = true;
+        }
+        catch {
+            rootExists = false;
+        }
+        if (!rootExists)
             return totals;
         const relativeBase = opts.relativeBase ?? this.config.wikiDir;
         const excludeFiles = new Set(opts.excludeFiles ?? []);
         const files = [];
-        const walk = (dirPath) => {
+        const walk = async (dirPath) => {
             let entries;
             try {
-                entries = fs.readdirSync(dirPath, { withFileTypes: true });
+                entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
             }
             catch (err) {
                 console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
                 return;
             }
+            // Sort for deterministic traversal order
+            entries.sort((a, b) => a.name.localeCompare(b.name));
             for (const entry of entries) {
                 const entryPath = path.join(dirPath, entry.name);
                 if (entry.isDirectory()) {
-                    walk(entryPath);
+                    await walk(entryPath);
                     continue;
                 }
                 if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
@@ -100,7 +130,7 @@ export class MemoryIndexer {
                 }
             }
         };
-        walk(rootDir);
+        await walk(rootDir);
         const currentSources = new Set(files.map((file) => path.relative(relativeBase, file)));
         // Remove stale sources (deleted files or pre-migration daily-file sources)
         for (const stale of this.store.indexedSources()) {
@@ -109,8 +139,9 @@ export class MemoryIndexer {
                 totals.deleted += 1;
             }
         }
-        for (const file of files) {
-            const stats = await this.indexFileRelativeTo(file, relativeBase);
+        const preparedMutations = await mapLimit(files, opts.fileConcurrency ?? 2, (file) => this.prepareIndexFileRelativeTo(file, relativeBase));
+        for (const mutation of preparedMutations) {
+            const stats = this.commitPreparedMutation(mutation);
             totals.indexed += stats.indexed;
             totals.deleted += stats.deleted;
             totals.skipped += stats.skipped;
@@ -118,99 +149,38 @@ export class MemoryIndexer {
         return totals;
     }
     async search(query, topK) {
-        // Task 2.2: overfetch to supply enough distinct sources after per-source dedup
         const fetchLimit = topK * FETCH_MULTIPLIER;
-        let denseResults = [];
+        let embedding;
         if (this.store.vecAvailable) {
             try {
-                const queryEmbedding = await this.embedder.embed([query]);
-                denseResults = this.store.searchDense(queryEmbedding[0], fetchLimit);
+                const embeddings = await this.embedder.embed([query]);
+                embedding = embeddings[0];
             }
             catch (err) {
                 console.warn('[memory-indexer] Dense search embedding failed:', err);
             }
         }
-        const bm25Results = this.store.searchBm25(query, fetchLimit);
-        // Task 2.1: candidate set = dense ∪ bm25 — no intersection filter
-        const denseIds = new Set(denseResults.map((r) => r.id));
-        const bm25Ids = new Set(bm25Results.map((r) => r.id));
-        const unionIds = new Set([...denseIds, ...bm25Ids]);
-        // Task 2.5: preserve raw dense retrieval scores [0,1] for the precision floor check
-        const denseRetrievalScore = new Map(denseResults.map((r) => [r.id, r.score]));
-        // Task 2.3: fetch all union candidates once — supplies recency mtime AND final content (no second DB call)
-        const unionArray = [...unionIds];
-        const chunks = this.store.getChunksByIds(unionArray);
-        const chunkMap = new Map(chunks.map((c) => [c.id, c]));
-        // Task 2.3: recency channel — rank union by fileMtimeAt DESC; stable tiebreak on union insertion order
-        const unionIndexOf = new Map(unionArray.map((id, idx) => [id, idx]));
-        const recencyOrdered = [...unionArray].sort((a, b) => {
-            const mtimeA = chunkMap.get(a)?.fileMtimeAt ?? 0;
-            const mtimeB = chunkMap.get(b)?.fileMtimeAt ?? 0;
-            if (mtimeB !== mtimeA)
-                return mtimeB - mtimeA;
-            return (unionIndexOf.get(a) ?? 0) - (unionIndexOf.get(b) ?? 0);
-        });
-        // RRF score accumulation: dense + bm25 + weighted recency channels
-        const scoreMap = new Map();
-        denseResults.forEach((r, rank) => {
-            scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0, recency: 0 });
-        });
-        bm25Results.forEach((r, rank) => {
-            const entry = scoreMap.get(r.id) ?? { dense: 0, bm25: 0, recency: 0 };
-            entry.bm25 = 1 / (RRF_K + rank + 1);
-            scoreMap.set(r.id, entry);
-        });
-        recencyOrdered.forEach((id, rank) => {
-            const entry = scoreMap.get(id);
-            if (!entry)
-                return;
-            entry.recency = RECENCY_WEIGHT * (1 / (RRF_K + rank + 1));
+        const rows = this.store.searchHybrid({
+            query,
+            embedding,
+            topK,
+            fetchLimit,
+            denseScoreFloor: DENSE_SCORE_FLOOR,
+            recencyWeight: RECENCY_WEIGHT,
+            rrfK: RRF_K,
         });
-        // Task 2.4: active-channel normalization — maxScore sums weights of channels that produced ≥1 result
-        const denseActive = denseResults.length > 0;
-        const bm25Active = bm25Results.length > 0;
-        const recencyActive = unionIds.size > 0;
-        const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? RECENCY_WEIGHT : 0)) / (RRF_K + 1);
-        // Task 2.5: build ranked list, dropping dense-only candidates below the precision floor
-        const ranked = [...scoreMap.entries()]
-            .map(([id, scores]) => ({
-            id,
-            totalScore: scores.dense + scores.bm25 + scores.recency,
-            hasDense: scores.dense > 0,
-            hasBm25: scores.bm25 > 0,
-            isDenseOnly: denseIds.has(id) && !bm25Ids.has(id),
-        }))
-            .filter((candidate) => {
-            if (!candidate.isDenseOnly)
-                return true;
-            const retrieval = denseRetrievalScore.get(candidate.id);
-            // Only drop when a dense retrieval score exists and is below the floor
-            if (retrieval === undefined)
-                return true;
-            return retrieval >= DENSE_SCORE_FLOOR;
-        })
-            .sort((a, b) => b.totalScore - a.totalScore);
+        if (rows.length === 0)
+            return [];
         const results = [];
-        const seenSources = new Set();
-        for (const r of ranked) {
-            if (results.length >= topK)
-                break;
-            const chunk = chunkMap.get(r.id);
-            if (!chunk)
-                continue;
-            if (seenSources.has(chunk.source))
-                continue;
-            seenSources.add(chunk.source);
-            const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
-            const retriever = r.hasDense && r.hasBm25 ? 'both' : r.hasDense ? 'dense' : 'bm25';
+        for (const row of rows) {
             let contentSource = 'file';
             try {
-                chunk.content = fs.readFileSync(path.join(this.config.wikiDir, chunk.source), 'utf8');
+                row.chunk.content = await fs.promises.readFile(path.join(this.config.wikiDir, row.chunk.source), 'utf8');
             }
             catch {
                 contentSource = 'fallback';
             }
-            results.push({ chunk, score: normalizedScore, retriever, contentSource });
+            results.push({ chunk: row.chunk, score: row.score, retriever: row.retriever, contentSource });
         }
         return results;
     }
@@ -219,13 +189,13 @@ export class MemoryIndexer {
         const rawDir = path.join(this.config.wikiDir, 'raw');
         const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
         const lockPath = `${savePath}.lock`;
-        fs.mkdirSync(rawDir, { recursive: true });
+        await fs.promises.mkdir(rawDir, { recursive: true });
         const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
         if (!acquired) {
             console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
         }
         try {
-            fs.appendFileSync(savePath, `\n${content}\n`, 'utf8');
+            await fs.promises.appendFile(savePath, `\n${content}\n`, 'utf8');
         }
         finally {
             if (acquired)