@devflow-tools/knowledge-engine 0.2.3 → 0.3.1

package/dist/bm25.d.ts

@@ -0,0 +1,14 @@
+interface ChunkDoc {
+    id: string;
+    title: string;
+    content: string;
+}
+interface SearchResult {
+    id: string;
+    title: string;
+    content: string;
+    score: number;
+}
+export declare function bm25Search(query: string, chunks: ChunkDoc[], limit: number): SearchResult[];
+export {};
+//# sourceMappingURL=bm25.d.ts.map

package/dist/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAGA,UAAU,QAAQ;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,YAAY;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAsBD,wBAAgB,UAAU,CACxB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,QAAQ,EAAE,EAClB,KAAK,EAAE,MAAM,GACZ,YAAY,EAAE,CAsChB"}

package/dist/bm25.js

@@ -0,0 +1,58 @@
+const k1 = 1.2;
+const b = 0.75;
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9一-鿿\s]/g, " ")
+        .split(/\s+/)
+        .filter(t => t.length > 0);
+}
+function computeIdf(term, docs) {
+    const docCount = docs.length;
+    const termDocs = docs.filter(doc => doc.includes(term)).length;
+    if (termDocs === 0)
+        return 0;
+    return Math.log((docCount - termDocs + 0.5) / (termDocs + 0.5) + 1);
+}
+function avgDocLength(docs) {
+    if (docs.length === 0)
+        return 1;
+    return docs.reduce((sum, d) => sum + d.length, 0) / docs.length;
+}
+export function bm25Search(query, chunks, limit) {
+    if (chunks.length === 0 || query.trim().length === 0)
+        return [];
+    const docs = chunks.map(c => tokenize(c.content + " " + c.title));
+    const avgdl = avgDocLength(docs);
+    const queryTerms = tokenize(query);
+    const results = chunks
+        .map((chunk, i) => {
+        const doc = docs[i];
+        const docLen = doc.length;
+        const score = queryTerms.reduce((sum, term) => {
+            const idf = computeIdf(term, docs);
+            let tf = 0;
+            for (const t of doc) {
+                if (t === term)
+                    tf++;
+            }
+            const numerator = tf * (k1 + 1);
+            const denominator = tf + k1 * (1 - b + b * (docLen / avgdl));
+            return sum + idf * (numerator / denominator);
+        }, 0);
+        // Boost exact title match
+        const titleTokens = tokenize(chunk.title);
+        const titleBoost = titleTokens.some(t => queryTerms.includes(t)) ? 1.5 : 1.0;
+        return {
+            id: chunk.id,
+            title: chunk.title,
+            content: chunk.content.slice(0, 500),
+            score: score * titleBoost,
+        };
+    })
+        .filter(r => r.score > 0)
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    return results;
+}
+//# sourceMappingURL=bm25.js.map

package/dist/bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAef,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,IAAgB;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,IAAgB;IACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,MAAkB,EAClB,KAAa;IAEb,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACnC,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,IAAI;oBAAE,EAAE,EAAE,CAAC;YACvB,CAAC;YACD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YAC7D,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;QAC/C,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,0BAA0B;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAE7E,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACpC,KAAK,EAAE,KAAK,GAAG,UAAU;SAC1B,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEnB,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/fetcher.d.ts

@@ -0,0 +1,2 @@
+export declare function fetchHtml(url: string): Promise<string>;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAGA,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAqB5D"}

package/dist/fetcher.js

@@ -0,0 +1,25 @@
+const USER_AGENT = "DevFlow-KnowledgeSync/0.1 (Node.js)";
+const MAX_RETRIES = 1;
+export async function fetchHtml(url) {
+    let lastError = null;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        try {
+            const res = await fetch(url, {
+                headers: { "User-Agent": USER_AGENT },
+                signal: AbortSignal.timeout(30_000),
+            });
+            if (!res.ok) {
+                throw new Error(`Failed to fetch "${url}": ${res.status} ${res.statusText}`);
+            }
+            return await res.text();
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt < MAX_RETRIES) {
+                await new Promise(r => setTimeout(r, 1000));
+            }
+        }
+    }
+    throw lastError ?? new Error(`Failed to fetch "${url}": unknown error`);
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,GAAG,qCAAqC,CAAC;AACzD,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW;IACzC,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;gBACrC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAY,CAAC;YACzB,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,oBAAoB,GAAG,kBAAkB,CAAC,CAAC;AAC1E,CAAC"}

package/dist/index.d.ts

@@ -1,2 +1,5 @@
 export { KnowledgeEngine } from "./knowledge-engine.js";
+export { fetchHtml } from "./fetcher.js";
+export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
+export { bm25Search } from "./bm25.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -1,2 +1,5 @@
 export { KnowledgeEngine } from "./knowledge-engine.js";
+export { fetchHtml } from "./fetcher.js";
+export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
+export { bm25Search } from "./bm25.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/knowledge-engine.d.ts

@@ -1,14 +1,15 @@
-import type { IKnowledgeEngine, KnowledgeEntry, KnowledgeSource, SearchOptions, SyncResult, KnowledgeSourceStatus, IDataProvider } from "@devflow-tools/sdk";
-export declare class KnowledgeEngine implements IKnowledgeEngine {
-    private provider;
-    private sourcesKey;
-    constructor(provider: IDataProvider);
-    private loadSourceRegistry;
-    private saveSourceRegistry;
+import type { KnowledgeEntry, KnowledgeSource, SearchOptions, SyncResult, KnowledgeSourceStatus } from "@devflow-tools/sdk";
+export declare class KnowledgeEngine {
+    private projectRoot;
+    constructor(opts: {
+        projectRoot: string;
+    });
+    private get knowledgeDir();
+    private pluginDir;
     registerSource(source: KnowledgeSource): Promise<void>;
-    syncDocs(_pluginName?: string): Promise<SyncResult[]>;
+    syncDocs(pluginName?: string): Promise<SyncResult[]>;
     search(query: string, options?: SearchOptions): Promise<KnowledgeEntry[]>;
-    lookup(symbol: string, _scope?: string): Promise<KnowledgeEntry | null>;
+    lookup(symbol: string, scope?: string): Promise<KnowledgeEntry | null>;
     getSources(): Promise<KnowledgeSourceStatus[]>;
 }
 //# sourceMappingURL=knowledge-engine.d.ts.map

package/dist/knowledge-engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAAE,cAAc,EAAE,eAAe,EACjD,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,aAAa,EAChE,MAAM,oBAAoB,CAAC;AAE5B,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,UAAU,CAAuB;gBAE7B,QAAQ,EAAE,aAAa;YAIrB,kBAAkB;YAoBlB,kBAAkB;IAW1B,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAIrD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAiBzE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IASvE,UAAU,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;CAIrD"}
+{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAyB5H,qBAAa,eAAe;IAC1B,OAAO,CAAC,WAAW,CAAS;gBAEhB,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE;IAIzC,OAAO,KAAK,YAAY,GAEvB;IAED,OAAO,CAAC,SAAS;IAIX,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAetD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA2DpD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsEzE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAOtE,UAAU,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;CAiCrD"}

package/dist/knowledge-engine.js

@@ -1,64 +1,178 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { fetchHtml } from "./fetcher.js";
+import { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
+import { bm25Search } from "./bm25.js";
 export class KnowledgeEngine {
-    provider;
-    sourcesKey = "knowledge:sources";
-    constructor(provider) {
-        this.provider = provider;
+    projectRoot;
+    constructor(opts) {
+        this.projectRoot = opts.projectRoot;
     }
-    async loadSourceRegistry() {
-        const sources = new Map();
-        const raw = await this.provider.kvGet(this.sourcesKey);
-        if (raw) {
-            const data = raw;
-            for (const item of data) {
-                sources.set(item.name, {
-                    source: item.source,
-                    status: item.status,
-                    lastSyncTime: item.lastSyncTime,
-                    entryCount: item.entryCount,
-                });
-            }
-        }
-        return sources;
+    get knowledgeDir() {
+        return join(this.projectRoot, ".devflow", "knowledge");
     }
-    async saveSourceRegistry(sources) {
-        const data = Array.from(sources.entries()).map(([name, status]) => ({
-            name,
-            source: status.source,
-            status: status.status,
-            lastSyncTime: status.lastSyncTime,
-            entryCount: status.entryCount,
-        }));
-        await this.provider.kvSet(this.sourcesKey, data);
+    pluginDir(pluginName) {
+        return join(this.knowledgeDir, pluginName);
     }
     async registerSource(source) {
-        const sources = await this.loadSourceRegistry();
-        sources.set(source.name, { source, status: "idle" });
-        await this.saveSourceRegistry(sources);
+        const dir = this.pluginDir(source.name);
+        mkdirSync(dir, { recursive: true });
+        const manifestPath = join(dir, "manifest.json");
+        if (!existsSync(manifestPath)) {
+            writeFileSync(manifestPath, JSON.stringify({
+                urls: source.urls,
+                fetchedAt: 0,
+                chunkCount: 0,
+                pluginName: source.pluginName,
+                version: source.version,
+            }, null, 2), "utf-8");
+        }
     }
-    async syncDocs(_pluginName) {
-        return [];
+    async syncDocs(pluginName) {
+        const name = pluginName ?? "";
+        if (!name)
+            return [];
+        const results = [];
+        const dir = this.pluginDir(name);
+        mkdirSync(dir, { recursive: true });
+        const manifestPath = join(dir, "manifest.json");
+        if (!existsSync(manifestPath))
+            return [];
+        const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
+        const urls = manifest.urls ?? [];
+        for (const url of urls) {
+            const errors = [];
+            let added = 0;
+            try {
+                const html = await fetchHtml(url);
+                const markdown = htmlToMarkdown(html);
+                const chunks = sliceByHeadings(markdown);
+                const chunkFiles = chunks.map((chunk, i) => ({
+                    id: `chunk-${String(i + 1).padStart(3, "0")}`,
+                    title: chunk.title,
+                    section: new URL(url).pathname,
+                    content: chunk.content,
+                    sourceUrl: url,
+                    fetchedAt: Date.now(),
+                }));
+                for (const cf of chunkFiles) {
+                    writeFileSync(join(dir, `${cf.id}.json`), JSON.stringify(cf, null, 2), "utf-8");
+                }
+                added = chunkFiles.length;
+                writeFileSync(manifestPath, JSON.stringify({
+                    urls: manifest.urls,
+                    fetchedAt: Date.now(),
+                    chunkCount: chunkFiles.length,
+                    pluginName: manifest.pluginName,
+                    version: manifest.version,
+                    sourceUrl: url,
+                }, null, 2), "utf-8");
+            }
+            catch (err) {
+                errors.push(err.message);
+            }
+            results.push({ source: url, added, updated: 0, errors });
+        }
+        return results;
     }
     async search(query, options) {
         const limit = options?.limit ?? 10;
-        const rows = await this.provider.sqlQuery("SELECT id, source, title, content FROM devflow_knowledge_entries WHERE content LIKE ? OR title LIKE ? LIMIT ?", [`%${query}%`, `%${query}%`, limit]);
-        return rows.map((r) => ({
+        const sources = options?.sources;
+        if (!existsSync(this.knowledgeDir))
+            return [];
+        const allChunks = [];
+        const pluginDirs = sources ?? readdirSync(this.knowledgeDir, { withFileTypes: true })
+            .filter(d => d.isDirectory())
+            .map(d => d.name);
+        for (const pluginName of pluginDirs) {
+            const dir = this.pluginDir(pluginName);
+            if (!existsSync(dir))
+                continue;
+            // Read version from manifest
+            let version;
+            const manifestPath = join(dir, "manifest.json");
+            if (existsSync(manifestPath)) {
+                try {
+                    const m = JSON.parse(readFileSync(manifestPath, "utf-8"));
+                    version = m.version;
+                }
+                catch { /* ignore */ }
+            }
+            const files = readdirSync(dir).filter(f => f.startsWith("chunk-") && f.endsWith(".json"));
+            for (const file of files) {
+                try {
+                    const chunk = JSON.parse(readFileSync(join(dir, file), "utf-8"));
+                    allChunks.push({
+                        id: `${pluginName}:${chunk.id}`,
+                        title: chunk.title,
+                        content: chunk.content,
+                        sourceUrl: chunk.sourceUrl,
+                        pluginName,
+                        version,
+                    });
+                }
+                catch {
+                    // Skip corrupted files
+                }
+            }
+        }
+        if (query.trim().length === 0) {
+            return allChunks.slice(0, limit).map(c => ({
+                id: c.id,
+                source: c.pluginName,
+                title: c.title,
+                content: c.content.slice(0, 500),
+                url: c.sourceUrl,
+                version: c.version,
+                embedding: [],
+            }));
+        }
+        return bm25Search(query, allChunks, limit).map(r => ({
             id: r.id,
-            source: r.source,
+            source: allChunks.find(c => c.id === r.id)?.pluginName ?? "",
             title: r.title,
             content: r.content,
+            url: allChunks.find(c => c.id === r.id)?.sourceUrl,
+            version: allChunks.find(c => c.id === r.id)?.version,
             embedding: [],
         }));
     }
-    async lookup(symbol, _scope) {
-        const results = await this.search(symbol, { limit: 5 });
+    async lookup(symbol, scope) {
+        const results = await this.search(symbol, { limit: 5, sources: scope ? [scope] : undefined });
         if (results.length === 0)
             return null;
-        const exact = results.find((r) => r.title.toLowerCase().includes(symbol.toLowerCase()));
+        const exact = results.find(r => r.title.toLowerCase().includes(symbol.toLowerCase()));
         return exact ?? null;
     }
     async getSources() {
-        const sources = await this.loadSourceRegistry();
-        return Array.from(sources.values());
+        if (!existsSync(this.knowledgeDir))
+            return [];
+        const pluginDirs = readdirSync(this.knowledgeDir, { withFileTypes: true })
+            .filter(d => d.isDirectory())
+            .map(d => d.name);
+        return pluginDirs.map(name => {
+            const manifestPath = join(this.pluginDir(name), "manifest.json");
+            const hasManifest = existsSync(manifestPath);
+            let chunkCount = 0;
+            let lastSync = 0;
+            let urls = [];
+            let pluginName = "";
+            let version = "";
+            if (hasManifest) {
+                const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
+                chunkCount = manifest.chunkCount ?? 0;
+                lastSync = manifest.fetchedAt ?? 0;
+                urls = manifest.urls ?? [];
+                pluginName = manifest.pluginName ?? "";
+                version = manifest.version ?? "";
+            }
+            return {
+                source: { name, urls, pluginName, version },
+                status: chunkCount > 0 ? "synced" : "idle",
+                lastSyncTime: lastSync || undefined,
+                entryCount: chunkCount || undefined,
+            };
+        });
     }
 }
 //# sourceMappingURL=knowledge-engine.js.map

package/dist/knowledge-engine.js.map

@@ -1 +1 @@
-{"version":3,"file":"knowledge-engine.js","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,eAAe;IAClB,QAAQ,CAAgB;IACxB,UAAU,GAAG,mBAAmB,CAAC;IAEzC,YAAY,QAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACzD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvD,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,GAGX,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAyC;oBACtD,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,OAA2C;QAC1E,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YAClE,IAAI;YACJ,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAuB;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,WAAoB;QACjC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAuB;QACjD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAGvC,+GAA+G,EAC/G,CAAC,IAAI,KAAK,GAAG,EAAE,IAAI,KAAK,GAAG,EAAE,KAAK,CAAC,CACpC,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,MAAe;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CACrD,CAAC;QACF,OAAO,KAAK,IAAI,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;CACF"}
+{"version":3,"file":"knowledge-engine.js","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACrF,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAoBvC,MAAM,OAAO,eAAe;IAClB,WAAW,CAAS;IAE5B,YAAY,IAA6B;QACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACtC,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IACzD,CAAC;IAEO,SAAS,CAAC,UAAkB;QAClC,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAuB;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9B,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC;gBACzC,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAmB;QAChC,MAAM,IAAI,GAAG,UAAU,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzC,MAAM,QAAQ,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAa,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;gBAEzC,MAAM,UAAU,GAAgB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACxD,EAAE,EAAE,SAAS,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;oBAC7C,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;oBAC9B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,SAAS,EAAE,GAAG;oBACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAC,CAAC;gBAEJ,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;oBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAClF,CAAC;gBAED,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;gBAE1B,aAAa,CACX,YAAY,EACZ,IAAI,CAAC,SAAS,CAAC;oBACb,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,UAAU,EAAE,UAAU,CAAC,MAAM;oBAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,SAAS,EAAE,GAAG;iBACf,EAAE,IAAI,EAAE,CAAC,CAAC,EACX,OAAO,CACR,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAuB;QACjD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAEjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,SAAS,GAGV,EAAE,CAAC;QAER,MAAM,UAAU,GAAG,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAClF,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEpB,KAAK,MAAM,UAAU,IAAI,UAAU,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE/B,6BAA6B;YAC7B,IAAI,OAA2B,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAChD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,CAAC,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;oBACxE,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;gBACtB,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,KAAK,GAAc,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;oBAC5E,SAAS,CAAC,IAAI,CAAC;wBACb,EAAE,EAAE,GAAG,UAAU,IAAI,KAAK,CAAC,EAAE,EAAE;wBAC/B,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,UAAU;wBACV,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,uBAAuB;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,MAAM,EAAE,CAAC,CAAC,UAAU;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAChC,GAAG,EAAE,CAAC,CAAC,SAAS;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,SAAS,EAAE,EAAE;aACd,CAAC,CAAC,CAAC;QACN,CAAC;QAED,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,IAAI,EAAE;YAC5D,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS;YAClD,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO;YACpD,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,KAAc;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACtF,OAAO,KAAK,IAAI,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACvE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEpB,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;YAC7C,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,IAAI,GAAa,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,EAAE,CAAC;YACpB,IAAI,OAAO,GAAG,EAAE,CAAC;YAEjB,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/E,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;gBACtC,QAAQ,GAAG,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACnC,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC3B,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;gBACvC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;YACnC,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE;gBAC3C,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAiB,CAAC,CAAC,CAAC,MAAe;gBAC5D,YAAY,EAAE,QAAQ,IAAI,SAAS;gBACnC,UAAU,EAAE,UAAU,IAAI,SAAS;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/markdown-converter.d.ts

@@ -0,0 +1,7 @@
+export declare function htmlToMarkdown(html: string): string;
+export interface Chunk {
+    title: string;
+    content: string;
+}
+export declare function sliceByHeadings(markdown: string): Chunk[];
+//# sourceMappingURL=markdown-converter.d.ts.map

package/dist/markdown-converter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-converter.d.ts","sourceRoot":"","sources":["../src/markdown-converter.ts"],"names":[],"mappings":"AAUA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,MAAM,WAAW,KAAK;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,KAAK,EAAE,CAmDzD"}

package/dist/markdown-converter.js

@@ -0,0 +1,61 @@
+import TurndownService from "turndown";
+const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+});
+// Remove script/style tags before conversion
+turndown.remove(["script", "style"]);
+export function htmlToMarkdown(html) {
+    return turndown.turndown(html);
+}
+export function sliceByHeadings(markdown) {
+    const trimmed = markdown.trim();
+    if (trimmed.length === 0)
+        return [];
+    const lines = trimmed.split("\n");
+    const chunks = [];
+    let currentTitle = "";
+    let currentContent = [];
+    let preambleContent = [];
+    let foundFirstH2 = false;
+    for (const line of lines) {
+        if (line.startsWith("## ")) {
+            if (foundFirstH2) {
+                // Push previous H2 section
+                if (currentContent.length > 0) {
+                    chunks.push({ title: currentTitle, content: currentContent.join("\n").trim() });
+                }
+            }
+            else {
+                // This is the first H2 encountered
+                foundFirstH2 = true;
+            }
+            currentTitle = line.slice(3).trim();
+            // Prepend preamble content to the first H2 section
+            currentContent = preambleContent.length > 0 ? [...preambleContent] : [];
+            preambleContent = [];
+        }
+        else if (!foundFirstH2) {
+            preambleContent.push(line);
+        }
+        else {
+            currentContent.push(line);
+        }
+    }
+    // Push final H2 section
+    if (currentContent.some(l => l.trim().length > 0)) {
+        chunks.push({ title: currentTitle, content: currentContent.join("\n").trim() });
+    }
+    // If no H2 headings at all, create one chunk from entire markdown
+    if (!foundFirstH2) {
+        const h1Match = trimmed.match(/^# (.+)$/m);
+        return [{ title: h1Match?.[1] ?? "Untitled", content: trimmed }];
+    }
+    // If preamble exists but no H2 sections were created (empty H2 sections)
+    if (chunks.length === 0 && preambleContent.length > 0) {
+        const h1Match = trimmed.match(/^# (.+)$/m);
+        return [{ title: h1Match?.[1] ?? "Untitled", content: trimmed }];
+    }
+    return chunks;
+}
+//# sourceMappingURL=markdown-converter.js.map

package/dist/markdown-converter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-converter.js","sourceRoot":"","sources":["../src/markdown-converter.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;IACnC,YAAY,EAAE,KAAK;IACnB,cAAc,EAAE,QAAQ;CACzB,CAAC,CAAC;AAEH,6CAA6C;AAC7C,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;AAErC,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAOD,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,cAAc,GAAa,EAAE,CAAC;IAClC,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,YAAY,EAAE,CAAC;gBACjB,2BAA2B;gBAC3B,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClF,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,mCAAmC;gBACnC,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;YACD,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACpC,mDAAmD;YACnD,cAAc,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,eAAe,GAAG,EAAE,CAAC;QACvB,CAAC;aAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YACzB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,kEAAkE;IAClE,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,yEAAyE;IACzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/knowledge-engine",
-  "version": "0.2.3",
+  "version": "0.3.1",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -17,14 +17,16 @@
     "clean": "rm -rf dist"
   },
   "dependencies": {
-    "@devflow-tools/sdk": "^0.2.3"
+    "@devflow-tools/sdk": "^0.3.0",
+    "turndown": "^7.2.4"
   },
   "devDependencies": {
+    "@types/turndown": "^5.0.6",
     "typescript": "^5.5.0",
     "vitest": "^2.0.0"
   },
   "files": [
     "dist"
   ],
-  "gitHead": "9951e2306d1a742cdecc0868154fd3f74d6d2684"
+  "gitHead": "2d52bb1401b6501a527cac7de0f664c18bf60dcf"
 }