@hanhnd/agent-kit 1.0.27 → 1.0.29

package/dist/memory/chunker.js

@@ -0,0 +1,147 @@
+import { createHash } from 'crypto';
+export function computeChunkId(content) {
+    return createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+export function cleanContentForEmbedding(text) {
+    return text.replace(/<!--[\s\S]*?-->/g, '');
+}
+const HEADING_RE = /^(#{1,6}) (.+)$/;
+const SENTENCE_END_RE = /([.!?。！？])\s+(?![0-9])/g;
+const URL_RE = /https?:\/\/\S+/g;
+function splitAtParagraph(text, maxLen) {
+    const paragraphs = text.split(/\n\n+/);
+    const parts = [];
+    let current = '';
+    for (const para of paragraphs) {
+        const candidate = current ? `${current}\n\n${para}` : para;
+        if (candidate.length <= maxLen) {
+            current = candidate;
+        }
+        else {
+            if (current)
+                parts.push(current);
+            current = para.length <= maxLen ? para : splitAtLine(para, maxLen).join('\n\n');
+        }
+    }
+    if (current)
+        parts.push(current);
+    return parts.filter(Boolean);
+}
+function splitAtLine(text, maxLen) {
+    const lines = text.split('\n');
+    const parts = [];
+    let current = '';
+    for (const line of lines) {
+        const candidate = current ? `${current}\n${line}` : line;
+        if (candidate.length <= maxLen) {
+            current = candidate;
+        }
+        else {
+            if (current)
+                parts.push(current);
+            current = line.length <= maxLen ? line : splitAtSentence(line, maxLen).join(' ');
+        }
+    }
+    if (current)
+        parts.push(current);
+    return parts.filter(Boolean);
+}
+function splitAtSentence(text, maxLen) {
+    const sanitized = text.replace(URL_RE, '[URL]');
+    const sentences = [];
+    let lastIndex = 0;
+    for (const match of sanitized.matchAll(SENTENCE_END_RE)) {
+        const end = (match.index ?? 0) + match[0].length;
+        sentences.push(text.slice(lastIndex, end).trim());
+        lastIndex = end;
+    }
+    if (lastIndex < text.length) {
+        sentences.push(text.slice(lastIndex).trim());
+    }
+    if (sentences.length === 0)
+        return [text.slice(0, maxLen)];
+    const parts = [];
+    let current = '';
+    for (const sentence of sentences) {
+        const candidate = current ? `${current} ${sentence}` : sentence;
+        if (candidate.length <= maxLen) {
+            current = candidate;
+        }
+        else {
+            if (current)
+                parts.push(current);
+            current = sentence.slice(0, maxLen);
+        }
+    }
+    if (current)
+        parts.push(current);
+    return parts.filter(Boolean);
+}
+function splitSection(text, maxLen) {
+    if (text.length <= maxLen)
+        return [text];
+    const byPara = splitAtParagraph(text, maxLen);
+    if (byPara.every((p) => p.length <= maxLen))
+        return byPara;
+    return byPara.flatMap((p) => (p.length <= maxLen ? [p] : splitAtLine(p, maxLen)));
+}
+export function chunkMarkdown(text, source, config) {
+    if (!text || !text.trim())
+        return [];
+    const { chunkSize, overlapLines } = config;
+    const lines = text.split('\n');
+    const sections = [];
+    let current = { heading: '', headingLevel: 0, lines: [] };
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const headingMatch = line.match(HEADING_RE);
+        if (headingMatch) {
+            if (current.lines.length > 0)
+                sections.push(current);
+            current = {
+                heading: headingMatch[2].trim(),
+                headingLevel: headingMatch[1].length,
+                lines: [{ text: line, lineNum: i + 1 }],
+            };
+        }
+        else {
+            current.lines.push({ text: line, lineNum: i + 1 });
+        }
+    }
+    if (current.lines.length > 0)
+        sections.push(current);
+    const chunks = [];
+    let prevOverlapLines = [];
+    for (const section of sections) {
+        const sectionText = section.lines.map((l) => l.text).join('\n');
+        if (!sectionText.trim())
+            continue;
+        const fullText = prevOverlapLines.length > 0
+            ? `${prevOverlapLines.join('\n')}\n${sectionText}`
+            : sectionText;
+        const parts = splitSection(fullText, chunkSize);
+        for (let pi = 0; pi < parts.length; pi++) {
+            const part = parts[pi];
+            if (!part.trim())
+                continue;
+            const partLines = part.split('\n');
+            const firstLine = section.lines[0]?.lineNum ?? 1;
+            const lastLine = section.lines[section.lines.length - 1]?.lineNum ?? firstLine;
+            const content = cleanContentForEmbedding(part);
+            chunks.push({
+                id: computeChunkId(content),
+                source,
+                heading: section.heading,
+                headingLevel: section.headingLevel,
+                content,
+                lineStart: firstLine,
+                lineEnd: lastLine,
+            });
+            // Update overlap for next iteration (last N lines of this chunk)
+            if (pi === parts.length - 1) {
+                prevOverlapLines = partLines.slice(-overlapLines);
+            }
+        }
+    }
+    return chunks;
+}

package/dist/memory/embedder.d.ts

@@ -0,0 +1,13 @@
+export declare class EmbedderError extends Error {
+    constructor(message: string, cause?: unknown);
+}
+export declare class Embedder {
+    private readonly modelName;
+    private pipeline;
+    private _dimension;
+    constructor(modelName: string);
+    embed(texts: string[]): Promise<Float32Array[]>;
+    get dimension(): number | undefined;
+    isReady(): boolean;
+    private loadModel;
+}

package/dist/memory/embedder.js

@@ -0,0 +1,68 @@
+import * as os from 'os';
+import * as path from 'path';
+const DEFAULT_CACHE_DIR = path.join(os.homedir(), '.cache', 'fastembed');
+export class EmbedderError extends Error {
+    constructor(message, cause) {
+        super(message);
+        this.name = 'EmbedderError';
+        if (cause instanceof Error)
+            this.cause = cause;
+    }
+}
+export class Embedder {
+    modelName;
+    pipeline = null;
+    _dimension;
+    constructor(modelName) {
+        this.modelName = modelName;
+    }
+    async embed(texts) {
+        if (texts.length === 0)
+            return [];
+        if (!this.pipeline) {
+            await this.loadModel();
+        }
+        try {
+            const results = await this.pipeline(texts);
+            return results.map((vec) => {
+                const arr = new Float32Array(vec);
+                if (!this._dimension)
+                    this._dimension = arr.length;
+                return arr;
+            });
+        }
+        catch (cause) {
+            throw new EmbedderError(`Embedding failed for batch of ${texts.length} texts`, cause);
+        }
+    }
+    get dimension() {
+        return this._dimension;
+    }
+    isReady() {
+        return this.pipeline !== null;
+    }
+    async loadModel() {
+        try {
+            const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
+            const modelMap = {
+                'Xenova/bge-small-en-v1.5': EmbeddingModel.BGESmallENV15,
+                'Xenova/bge-base-en-v1.5': EmbeddingModel.BGEBaseENV15,
+                'Xenova/all-MiniLM-L6-v2': EmbeddingModel.AllMiniLML6V2,
+            };
+            const model = modelMap[this.modelName] ?? EmbeddingModel.BGESmallENV15;
+            const embedder = await FlagEmbedding.init({ model: model, cacheDir: DEFAULT_CACHE_DIR, showDownloadProgress: false });
+            this.pipeline = async (texts) => {
+                const result = [];
+                for await (const batch of embedder.embed(texts, texts.length)) {
+                    for (const vec of batch) {
+                        result.push(Array.from(vec));
+                    }
+                }
+                return result;
+            };
+        }
+        catch (cause) {
+            throw new EmbedderError(`Failed to load embedding model: ${this.modelName}`, cause);
+        }
+    }
+}

package/dist/memory/index.d.ts

@@ -0,0 +1,5 @@
+export { MemoryStore, StoreError } from './store.js';
+export { Embedder, EmbedderError } from './embedder.js';
+export { MemoryIndexer } from './indexer.js';
+export type { MemoryChunk, SearchResult, MemoryConfig, IndexStats } from './types.js';
+export { DEFAULT_MEMORY_CONFIG } from './types.js';

package/dist/memory/index.js

@@ -0,0 +1,4 @@
+export { MemoryStore, StoreError } from './store.js';
+export { Embedder, EmbedderError } from './embedder.js';
+export { MemoryIndexer } from './indexer.js';
+export { DEFAULT_MEMORY_CONFIG } from './types.js';

package/dist/memory/indexer.d.ts

@@ -0,0 +1,20 @@
+import type { Embedder } from './embedder.js';
+import type { MemoryStore } from './store.js';
+import type { IndexStats, MemoryConfig, SearchResult } from './types.js';
+export declare class MemoryIndexer {
+    private readonly store;
+    private readonly embedder;
+    private readonly config;
+    private _ready;
+    constructor(store: MemoryStore, embedder: Embedder, config: MemoryConfig);
+    indexFile(absolutePath: string): Promise<IndexStats>;
+    private indexFileRelativeTo;
+    indexDirectory(rootDir: string, opts?: {
+        relativeBase?: string;
+        excludeFiles?: string[];
+    }): Promise<IndexStats>;
+    search(query: string, topK: number): Promise<SearchResult[]>;
+    save(content: string): Promise<IndexStats>;
+    startupIndex(): Promise<void>;
+    get ready(): boolean;
+}

package/dist/memory/indexer.js

@@ -0,0 +1,222 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { chunkMarkdown } from './chunker.js';
+const LOCK_RETRY_MS = 50;
+const LOCK_TIMEOUT_MS = 500;
+async function acquireLock(lockPath) {
+    const deadline = Date.now() + LOCK_TIMEOUT_MS;
+    while (Date.now() < deadline) {
+        try {
+            fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' });
+            return true;
+        }
+        catch {
+            await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
+        }
+    }
+    return false;
+}
+function releaseLock(lockPath) {
+    try {
+        fs.unlinkSync(lockPath);
+    }
+    catch {
+        // ignore
+    }
+}
+export class MemoryIndexer {
+    store;
+    embedder;
+    config;
+    _ready = false;
+    constructor(store, embedder, config) {
+        this.store = store;
+        this.embedder = embedder;
+        this.config = config;
+    }
+    async indexFile(absolutePath) {
+        return this.indexFileRelativeTo(absolutePath, this.config.wikiDir);
+    }
+    async indexFileRelativeTo(absolutePath, sourceRoot) {
+        const source = path.relative(sourceRoot, absolutePath);
+        const existingHashes = this.store.hashesBySource(source);
+        let text;
+        try {
+            text = fs.readFileSync(absolutePath, 'utf8');
+        }
+        catch (err) {
+            console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
+            return { indexed: 0, deleted: 0, skipped: existingHashes.size };
+        }
+        const allChunks = chunkMarkdown(text, source, this.config);
+        const newHashes = new Set(allChunks.map((c) => c.id));
+        const toIndex = allChunks.filter((c) => !existingHashes.has(c.id));
+        const toDelete = [...existingHashes].filter((h) => !newHashes.has(h));
+        let embeddings = [];
+        if (toIndex.length > 0) {
+            try {
+                embeddings = await this.embedder.embed(toIndex.map((c) => c.content));
+            }
+            catch (err) {
+                console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
+                return { indexed: 0, deleted: 0, skipped: existingHashes.size };
+            }
+        }
+        if (toIndex.length > 0) {
+            this.store.upsert(toIndex, embeddings);
+        }
+        if (toDelete.length > 0) {
+            this.store.deleteByIds(toDelete);
+        }
+        return {
+            indexed: toIndex.length,
+            deleted: toDelete.length,
+            skipped: allChunks.length - toIndex.length,
+        };
+    }
+    async indexDirectory(rootDir, opts = {}) {
+        const totals = { indexed: 0, deleted: 0, skipped: 0 };
+        if (!fs.existsSync(rootDir))
+            return totals;
+        const relativeBase = opts.relativeBase ?? this.config.wikiDir;
+        const excludeFiles = new Set(opts.excludeFiles ?? []);
+        const files = [];
+        const walk = (dirPath) => {
+            let entries;
+            try {
+                entries = fs.readdirSync(dirPath, { withFileTypes: true });
+            }
+            catch (err) {
+                console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
+                return;
+            }
+            for (const entry of entries) {
+                const entryPath = path.join(dirPath, entry.name);
+                if (entry.isDirectory()) {
+                    walk(entryPath);
+                    continue;
+                }
+                if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
+                    files.push(entryPath);
+                }
+            }
+        };
+        walk(rootDir);
+        const currentSources = new Set(files.map((file) => path.relative(relativeBase, file)));
+        // Remove stale sources (deleted files or pre-migration daily-file sources)
+        for (const stale of this.store.indexedSources()) {
+            if (!currentSources.has(stale)) {
+                this.store.deleteBySource(stale);
+                totals.deleted += 1;
+            }
+        }
+        for (const file of files) {
+            const stats = await this.indexFileRelativeTo(file, relativeBase);
+            totals.indexed += stats.indexed;
+            totals.deleted += stats.deleted;
+            totals.skipped += stats.skipped;
+        }
+        return totals;
+    }
+    async search(query, topK) {
+        const fetchLimit = topK * 2;
+        let denseResults = [];
+        if (this.store.vecAvailable) {
+            try {
+                const queryEmbedding = await this.embedder.embed([query]);
+                denseResults = this.store.searchDense(queryEmbedding[0], fetchLimit);
+            }
+            catch (err) {
+                console.warn('[memory-indexer] Dense search embedding failed:', err);
+            }
+        }
+        const bm25Results = this.store.searchBm25(query, fetchLimit);
+        // RRF fusion (k=60)
+        const k = 60;
+        const scoreMap = new Map();
+        denseResults.forEach((r, rank) => {
+            scoreMap.set(r.id, { dense: 1 / (k + rank + 1), bm25: 0 });
+        });
+        bm25Results.forEach((r, rank) => {
+            const existing = scoreMap.get(r.id);
+            if (existing) {
+                existing.bm25 = 1 / (k + rank + 1);
+            }
+            else {
+                scoreMap.set(r.id, { dense: 0, bm25: 1 / (k + rank + 1) });
+            }
+        });
+        const hasDense = this.store.vecAvailable && denseResults.length > 0;
+        const numRetrievers = hasDense ? 2 : 1;
+        const maxScore = numRetrievers / 61;
+        const ranked = [...scoreMap.entries()]
+            .map(([id, scores]) => ({
+            id,
+            totalScore: scores.dense + scores.bm25,
+            hasDense: scores.dense > 0,
+            hasBm25: scores.bm25 > 0,
+        }))
+            .sort((a, b) => b.totalScore - a.totalScore);
+        const chunks = this.store.getChunksByIds(ranked.map((r) => r.id));
+        const chunkMap = new Map(chunks.map((c) => [c.id, c]));
+        const results = [];
+        const seenSources = new Set();
+        for (const r of ranked) {
+            if (results.length >= topK)
+                break;
+            const chunk = chunkMap.get(r.id);
+            if (!chunk)
+                continue;
+            if (seenSources.has(chunk.source))
+                continue;
+            seenSources.add(chunk.source);
+            const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
+            const retriever = r.hasDense && r.hasBm25 ? 'both' : r.hasDense ? 'dense' : 'bm25';
+            let contentSource = 'file';
+            try {
+                chunk.content = fs.readFileSync(path.join(this.config.wikiDir, chunk.source), 'utf8');
+            }
+            catch {
+                contentSource = 'fallback';
+            }
+            results.push({ chunk, score: normalizedScore, retriever, contentSource });
+        }
+        return results;
+    }
+    async save(content) {
+        const datePart = new Date().toISOString().slice(0, 10);
+        const rawDir = path.join(this.config.wikiDir, 'raw');
+        const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
+        const lockPath = `${savePath}.lock`;
+        fs.mkdirSync(rawDir, { recursive: true });
+        const acquired = await acquireLock(lockPath);
+        if (!acquired) {
+            console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
+        }
+        try {
+            fs.appendFileSync(savePath, `\n${content}\n`, 'utf8');
+        }
+        finally {
+            if (acquired)
+                releaseLock(lockPath);
+        }
+        return { indexed: 0, deleted: 0, skipped: 0 };
+    }
+    async startupIndex() {
+        try {
+            await this.indexDirectory(path.join(this.config.wikiDir, 'compiled'), {
+                relativeBase: this.config.wikiDir,
+                excludeFiles: ['index.md', 'log.md'],
+            });
+        }
+        catch (err) {
+            console.warn('[memory-indexer] Startup indexing failed:', err);
+        }
+        finally {
+            this._ready = true;
+        }
+    }
+    get ready() {
+        return this._ready;
+    }
+}

package/dist/memory/store.d.ts

@@ -0,0 +1,29 @@
+import type { MemoryChunk, MemoryConfig } from './types.js';
+export declare class StoreError extends Error {
+    constructor(message: string, cause?: unknown);
+}
+export declare class MemoryStore {
+    private readonly config;
+    private db;
+    private _vecAvailable;
+    constructor(dbPath: string, config: MemoryConfig);
+    private openDatabase;
+    private loadVecExtension;
+    private createSchema;
+    upsert(chunks: MemoryChunk[], embeddings: Float32Array[]): void;
+    hashesBySource(source: string): Set<string>;
+    indexedSources(): string[];
+    deleteBySource(source: string): void;
+    deleteByIds(ids: string[]): void;
+    searchDense(embedding: Float32Array, limit: number): Array<{
+        id: string;
+        score: number;
+    }>;
+    searchBm25(query: string, limit: number): Array<{
+        id: string;
+        score: number;
+    }>;
+    getChunksByIds(ids: string[]): MemoryChunk[];
+    get vecAvailable(): boolean;
+    close(): void;
+}