@kk-irving/knowledge-mcp-server 1.0.0

package/dist/aosp/indexer.js

@@ -0,0 +1,186 @@
+/**
+ * AOSP 模块索引器：遍历 module 路径下的源码 → 切块 → 入库。
+ *
+ * 设计：
+ *   - 输入 platform + module + module_path（可绝对，也可相对 repo_root） + repo_root
+ *   - 递归遍历，按 chunker 切块，写 aosp_chunks 表（UNIQUE 约束去重）
+ *   - 跳过黑名单目录、二进制、>5MB 文件
+ *   - 嵌入由后续 `embed_pending(source="aosp")` 异步处理
+ *
+ * 提供：
+ *   - indexAospModule({ platform, module, module_path, repo_root })
+ *   - clearAospIndex({ platform?, module? })
+ */
+import { mkdir, readdir, readFile, stat } from "node:fs/promises";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+import { getDb, runInTransaction } from "../db.js";
+import { chunkFile, shouldIndexFile, shouldSkipDir } from "./chunker.js";
+const UPSERT_SQL = `
+INSERT INTO aosp_chunks (
+  platform, module, module_path, file_path,
+  line_start, line_end, symbol_kind, symbol_name,
+  content, content_hash, indexed_at
+) VALUES (
+  ?, ?, ?, ?,
+  ?, ?, ?, ?,
+  ?, ?, datetime('now')
+)
+ON CONFLICT(platform, module, file_path, line_start, line_end) DO UPDATE SET
+  symbol_kind = excluded.symbol_kind,
+  symbol_name = excluded.symbol_name,
+  content = excluded.content,
+  content_hash = excluded.content_hash,
+  embedding = NULL,
+  embedding_updated_at = NULL,
+  indexed_at = excluded.indexed_at
+WHERE aosp_chunks.content_hash IS NOT excluded.content_hash
+`;
+export async function indexAospModule(args) {
+    const t0 = Date.now();
+    const stats = {
+        platform: args.platform,
+        module: args.module,
+        module_path: args.module_path,
+        files_scanned: 0,
+        files_indexed: 0,
+        chunks_inserted: 0,
+        chunks_unchanged: 0,
+        errors: 0,
+        elapsed_ms: 0,
+    };
+    if (!args.repo_root)
+        throw new Error("repo_root 不能为空");
+    await mkdir(args.repo_root, { recursive: true }).catch(() => { });
+    const absPath = path.isAbsolute(args.module_path)
+        ? args.module_path
+        : path.join(args.repo_root, args.module_path);
+    const stRoot = await stat(absPath).catch(() => null);
+    if (!stRoot)
+        throw new Error(`module_path 不存在: ${absPath}`);
+    if (!stRoot.isDirectory())
+        throw new Error(`module_path 不是目录: ${absPath}`);
+    const db = getDb();
+    const upsert = db.prepare(UPSERT_SQL);
+    // BFS 遍历
+    const queue = [absPath];
+    const fileRows = []; // [args[], hash]
+    while (queue.length > 0) {
+        const dir = queue.shift();
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            stats.errors++;
+            continue;
+        }
+        for (const entry of entries) {
+            const full = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (shouldSkipDir(entry.name))
+                    continue;
+                queue.push(full);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            stats.files_scanned++;
+            let st;
+            try {
+                st = await stat(full);
+            }
+            catch {
+                stats.errors++;
+                continue;
+            }
+            if (!shouldIndexFile(full, st.size))
+                continue;
+            let content;
+            try {
+                content = await readFile(full, "utf8");
+            }
+            catch {
+                stats.errors++;
+                continue;
+            }
+            // 跳过包含 NUL 的伪文本（疑似二进制）
+            if (content.indexOf("\u0000") >= 0)
+                continue;
+            const chunks = chunkFile(full, content);
+            if (chunks.length === 0)
+                continue;
+            stats.files_indexed++;
+            const relPath = path.relative(args.repo_root, full).replace(/\\/g, "/");
+            for (const ch of chunks) {
+                const hash = crypto
+                    .createHash("sha1")
+                    .update(ch.content)
+                    .digest("hex");
+                fileRows.push([
+                    [
+                        args.platform.toUpperCase(),
+                        args.module.toLowerCase(),
+                        args.module_path,
+                        relPath,
+                        String(ch.line_start),
+                        String(ch.line_end),
+                        ch.symbol_kind,
+                        ch.symbol_name,
+                        ch.content,
+                        hash,
+                    ],
+                    hash,
+                ]);
+            }
+        }
+    }
+    // 一次性事务写入
+    if (fileRows.length > 0) {
+        runInTransaction(db, () => {
+            for (const [args2, _hash] of fileRows) {
+                const info = upsert.run(...args2);
+                if ((info?.changes ?? 0) > 0)
+                    stats.chunks_inserted++;
+                else
+                    stats.chunks_unchanged++;
+            }
+        });
+    }
+    stats.elapsed_ms = Date.now() - t0;
+    return stats;
+}
+export function clearAospIndex(args = {}) {
+    const db = getDb();
+    const platform = args.platform?.toUpperCase();
+    const module = args.module?.toLowerCase();
+    let where = "";
+    const params = [];
+    if (platform && module) {
+        where = "WHERE platform = ? AND module = ?";
+        params.push(platform, module);
+    }
+    else if (platform) {
+        where = "WHERE platform = ?";
+        params.push(platform);
+    }
+    else if (module) {
+        where = "WHERE module = ?";
+        params.push(module);
+    }
+    const beforeRow = db
+        .prepare(`SELECT COUNT(*) AS n FROM aosp_chunks ${where}`)
+        .get(...params);
+    const before = beforeRow?.n ?? 0;
+    db.prepare(`DELETE FROM aosp_chunks ${where}`).run(...params);
+    return {
+        cleared: before,
+        scope: platform && module
+            ? `platform=${platform}, module=${module}`
+            : platform
+                ? `platform=${platform}`
+                : module
+                    ? `module=${module}`
+                    : "all",
+    };
+}

package/dist/aosp/module-map-loader.js

@@ -0,0 +1,176 @@
+/**
+ * 解析 steering/module-path-map.md，提取 D4/X5/STB 各平台的模块 → 路径映射。
+ *
+ * map 的 Markdown 结构（约定）：
+ *
+ * ```
+ * ## D4 平台
+ *
+ * ### 一级目录: vendor/zeasn
+ *
+ * | 模块 | 路径 |
+ * |---|---|
+ * | tvsystemui | `vendor/zeasn/...` |
+ * | asplayer | `vendor/zeasn/...` |
+ * ```
+ *
+ * 我们只关注**含路径的代码块或表格**，按平台 → module → [paths] 三层结构提取。
+ *
+ * 输出 `data/module-map.json`，下次启动直接读 JSON 不再解析 markdown。
+ */
+import { readFile, writeFile, mkdir, stat } from "node:fs/promises";
+import * as path from "node:path";
+import { existsSync } from "node:fs";
+import { config } from "../config.js";
+const PLATFORM_NAMES = ["D4", "X5", "STB"];
+/**
+ * 默认从仓库根的 `steering/module-path-map.md` 加载。
+ * 先试 cache: data/module-map.json；缺失或源文件更新过时则重新解析。
+ */
+export async function loadModuleMap(opts = {}) {
+    const sourcePath = opts.sourcePath ?? path.resolve("./steering/module-path-map.md");
+    const cachePath = opts.cachePath ?? path.resolve(path.dirname(config.dbPath), "module-map.json");
+    // 用 cache 如果存在且比源新
+    if (!opts.rebuild && existsSync(cachePath) && existsSync(sourcePath)) {
+        try {
+            const [cacheStat, srcStat] = await Promise.all([stat(cachePath), stat(sourcePath)]);
+            if (cacheStat.mtimeMs >= srcStat.mtimeMs) {
+                const text = await readFile(cachePath, "utf8");
+                return JSON.parse(text);
+            }
+        }
+        catch {
+            /* 落到重建 */
+        }
+    }
+    // 解析 markdown
+    if (!existsSync(sourcePath)) {
+        return {
+            platforms: {},
+            generated_at: new Date().toISOString(),
+            source: sourcePath,
+        };
+    }
+    const md = await readFile(sourcePath, "utf8");
+    const map = parseModulePathMap(md, sourcePath);
+    // 写 cache
+    try {
+        await mkdir(path.dirname(cachePath), { recursive: true });
+        await writeFile(cachePath, JSON.stringify(map, null, 2), "utf8");
+    }
+    catch {
+        /* cache 失败不致命 */
+    }
+    return map;
+}
+// =============================================================================
+// Markdown 解析
+// =============================================================================
+/**
+ * 解析 module-path-map.md 文本。
+ *
+ * 思路：
+ *   - 顺序扫描行，维护 currentPlatform / currentModule
+ *   - 平台 heading 形式：`## D4 平台` / `## X5 平台` / `## STB 平台`（或英文）
+ *   - 在表格行中提取 `| 模块名 | \`path/to/module\` |` 形式
+ *   - 在 inline code 块中提取 `` `vendor/...` `` 形式
+ *   - 模块名 token 化为 lowercase + 去空格用作 key
+ */
+export function parseModulePathMap(markdown, sourcePath) {
+    const lines = markdown.split(/\r?\n/);
+    const platforms = {};
+    let currentPlatform = null;
+    let currentModule = null;
+    const platformHeadingRe = /^##\s+(D4|X5|STB)\s*(?:平台)?/i;
+    const subHeadingRe = /^####\s+(.+?)\s*$/; // module 子节标题
+    const tableRowRe = /^\|\s*([^|]+?)\s*\|\s*`([^`]+)`/;
+    const codePathLineRe = /`([^`]+)`/g;
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        // 平台标题
+        const pm = line.match(platformHeadingRe);
+        if (pm) {
+            const name = pm[1].toUpperCase();
+            currentPlatform = PLATFORM_NAMES.includes(name) ? name : null;
+            currentModule = null;
+            if (currentPlatform && !platforms[currentPlatform])
+                platforms[currentPlatform] = {};
+            continue;
+        }
+        if (!currentPlatform)
+            continue;
+        // 模块子标题（如 `#### TvSystemUI`）
+        const sm = line.match(subHeadingRe);
+        if (sm) {
+            currentModule = normalizeModuleName(sm[1]);
+            if (currentModule && !platforms[currentPlatform][currentModule]) {
+                platforms[currentPlatform][currentModule] = [];
+            }
+            continue;
+        }
+        // 表格行：| 模块名 | `path` | ...
+        const trm = line.match(tableRowRe);
+        if (trm) {
+            // 跳过表头分隔符
+            if (/^[-:|\s]+$/.test(trm[1]))
+                continue;
+            // 跳过表头（"模块" / "路径" 这种无 path 的）
+            const moduleName = normalizeModuleName(trm[1]);
+            const pathStr = trm[2].trim();
+            if (moduleName && pathStr) {
+                if (!platforms[currentPlatform][moduleName])
+                    platforms[currentPlatform][moduleName] = [];
+                platforms[currentPlatform][moduleName].push(pathStr);
+            }
+            continue;
+        }
+        // 在 currentModule 段内的 inline code 路径
+        if (currentModule) {
+            let m;
+            const re = new RegExp(codePathLineRe.source, "g");
+            while ((m = re.exec(line)) !== null) {
+                const candidate = m[1].trim();
+                // 只接受形如 `xxx/yyy/...` 的路径片段
+                if (/^[\w._-]+\/[\w./_-]+/.test(candidate)) {
+                    const arr = platforms[currentPlatform][currentModule];
+                    if (arr && !arr.includes(candidate))
+                        arr.push(candidate);
+                }
+            }
+        }
+    }
+    return {
+        platforms,
+        generated_at: new Date().toISOString(),
+        source: sourcePath,
+    };
+}
+function normalizeModuleName(raw) {
+    const trimmed = raw
+        .replace(/[`*_]/g, "")
+        .trim()
+        .toLowerCase();
+    if (!trimmed)
+        return null;
+    if (/^(模块|路径|说明|备注|描述|—|-)/.test(trimmed))
+        return null;
+    // 把空格 / 标点替换成短横线
+    return trimmed.replace(/[\s,()（）/]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+}
+// =============================================================================
+// 查询帮手
+// =============================================================================
+/**
+ * 给定 platform + module，返回路径前缀数组（可能为空）。
+ */
+export function resolveModulePaths(map, platform, moduleId) {
+    const p = (platform || "").toUpperCase();
+    const m = (moduleId || "").toLowerCase();
+    return map.platforms[p]?.[m] ?? [];
+}
+export function listModulesOfPlatform(map, platform) {
+    const p = (platform || "").toUpperCase();
+    return Object.keys(map.platforms[p] ?? {}).sort();
+}

package/dist/aosp/search.js

@@ -0,0 +1,254 @@
+/**
+ * AOSP chunks 检索：vector / fts / hybrid 三模式 + platform/module 过滤。
+ *
+ * 与三源 search 共享 embedder 与 FTS5 设计，但单独存矩阵（aosp_chunks 通常远多于
+ * zmind_issues，可独立 invalidate）。
+ */
+import { config } from "../config.js";
+import { blobToVector, getDb } from "../db.js";
+import { embedOne } from "../embedder.js";
+import { loadModuleMap, resolveModulePaths } from "./module-map-loader.js";
+const _cache = new Map();
+function cacheKey(platform, module, modulePath) {
+    return `${(platform ?? "*").toUpperCase()}::${(module ?? "*").toLowerCase()}::${modulePath ?? ""}`;
+}
+export function invalidateAospIndex() {
+    _cache.clear();
+}
+function loadAospIndex(args) {
+    const key = cacheKey(args.platform, args.module, (args.modulePathPrefixes ?? []).join("|"));
+    const cached = _cache.get(key);
+    if (cached)
+        return cached;
+    const db = getDb();
+    const dim = config.embeddingDim;
+    const where = ["embedding IS NOT NULL"];
+    const params = [];
+    if (args.platform) {
+        where.push("platform = ?");
+        params.push(args.platform.toUpperCase());
+    }
+    if (args.module) {
+        where.push("module = ?");
+        params.push(args.module.toLowerCase());
+    }
+    if (args.modulePathPrefixes && args.modulePathPrefixes.length > 0) {
+        const ors = args.modulePathPrefixes.map(() => "module_path LIKE ?").join(" OR ");
+        where.push(`(${ors})`);
+        for (const p of args.modulePathPrefixes)
+            params.push(`${p}%`);
+    }
+    const rows = db
+        .prepare(`SELECT id, platform, module, module_path, file_path,
+              line_start, line_end, symbol_kind, symbol_name, content, embedding
+         FROM aosp_chunks
+        WHERE ${where.join(" AND ")}`)
+        .all(...params);
+    const matrix = new Float32Array(rows.length * dim);
+    const ids = [];
+    const meta = new Map();
+    rows.forEach((r, i) => {
+        if (!r.embedding)
+            return;
+        const vec = blobToVector(r.embedding, dim);
+        matrix.set(vec, i * dim);
+        ids.push(r.id);
+        meta.set(r.id, r);
+    });
+    const idx = { ids, matrix, count: rows.length, meta };
+    _cache.set(key, idx);
+    return idx;
+}
+// =============================================================================
+// FTS 检索
+// =============================================================================
+function ftsSearch(args) {
+    const db = getDb();
+    const safeQ = escapeFts5(args.query);
+    if (!safeQ)
+        return [];
+    const where = [];
+    const params = [];
+    where.push("aosp_chunks_fts MATCH ?");
+    params.push(safeQ);
+    if (args.platform) {
+        where.push("a.platform = ?");
+        params.push(args.platform.toUpperCase());
+    }
+    if (args.module) {
+        where.push("a.module = ?");
+        params.push(args.module.toLowerCase());
+    }
+    if (args.modulePathPrefixes && args.modulePathPrefixes.length > 0) {
+        const ors = args.modulePathPrefixes.map(() => "a.module_path LIKE ?").join(" OR ");
+        where.push(`(${ors})`);
+        for (const p of args.modulePathPrefixes)
+            params.push(`${p}%`);
+    }
+    const sql = `
+    SELECT a.id, a.platform, a.module, a.module_path, a.file_path,
+           a.line_start, a.line_end, a.symbol_kind, a.symbol_name, a.content,
+           bm25(aosp_chunks_fts) AS bm
+      FROM aosp_chunks_fts
+      JOIN aosp_chunks a ON a.id = aosp_chunks_fts.rowid
+     WHERE ${where.join(" AND ")}
+     ORDER BY bm
+     LIMIT ?
+  `;
+    const rows = db.prepare(sql).all(...params, args.limit);
+    return rows.map((r) => ({ row: r, bm25: r.bm }));
+}
+function escapeFts5(q) {
+    const tokens = q
+        .replace(/["']/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 0);
+    if (tokens.length === 0)
+        return "";
+    return tokens.map((t) => `"${t}"`).join(" ");
+}
+function snippetOf(content) {
+    const clean = content.replace(/\s+/g, " ").trim();
+    return clean.length <= config.snippetMaxChars ? clean : clean.slice(0, config.snippetMaxChars).trimEnd() + "…";
+}
+function rowToHit(r, score, match) {
+    return {
+        id: r.id,
+        source: "aosp",
+        platform: r.platform,
+        module: r.module,
+        file_path: r.file_path,
+        line_start: r.line_start,
+        line_end: r.line_end,
+        symbol_kind: r.symbol_kind,
+        symbol_name: r.symbol_name,
+        snippet: snippetOf(r.content),
+        score,
+        match,
+    };
+}
+function normalizeFts(values) {
+    const map = new Map();
+    if (values.length === 0)
+        return map;
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    const span = max - min || 1;
+    values.forEach((v) => map.set(v, 1 - (v - min) / span));
+    return map;
+}
+// =============================================================================
+// 公共入口
+// =============================================================================
+export async function searchAosp(args) {
+    const limit = Math.max(1, Math.min(config.searchMaxLimit, args.limit ?? config.searchDefaultLimit));
+    const mode = args.mode ?? "hybrid";
+    if (!args.query.trim()) {
+        return {
+            source: "aosp",
+            query: args.query,
+            mode,
+            hits: [],
+            filter: {},
+        };
+    }
+    // 把 module 翻译成 module_path 前缀
+    let modulePathPrefixes;
+    if (args.module_path) {
+        modulePathPrefixes = [args.module_path];
+    }
+    else if (args.module && args.platform) {
+        try {
+            const map = await loadModuleMap();
+            const paths = resolveModulePaths(map, args.platform, args.module);
+            if (paths.length > 0)
+                modulePathPrefixes = paths;
+        }
+        catch {
+            /* fallback to no prefix filter */
+        }
+    }
+    // FTS 部分
+    let ftsHits = [];
+    if (mode === "fts" || mode === "hybrid") {
+        ftsHits = ftsSearch({
+            query: args.query,
+            platform: args.platform,
+            module: args.module,
+            modulePathPrefixes,
+            limit: limit * 2,
+        });
+    }
+    // Vector 部分
+    let vectorHits = [];
+    if (mode === "vector" || mode === "hybrid") {
+        try {
+            const vec = await embedOne(args.query);
+            const idx = loadAospIndex({ platform: args.platform, module: args.module, modulePathPrefixes });
+            if (idx.count > 0) {
+                const dim = config.embeddingDim;
+                const scores = new Float32Array(idx.count);
+                for (let i = 0; i < idx.count; i++) {
+                    let s = 0;
+                    const off = i * dim;
+                    for (let j = 0; j < dim; j++)
+                        s += idx.matrix[off + j] * vec[j];
+                    scores[i] = s;
+                }
+                const indices = Array.from({ length: idx.count }, (_, k) => k).sort((a, b) => scores[b] - scores[a]);
+                for (let i = 0; i < Math.min(limit * 2, indices.length); i++) {
+                    const id = idx.ids[indices[i]];
+                    const row = idx.meta.get(id);
+                    if (row)
+                        vectorHits.push({ row, score: scores[indices[i]] });
+                }
+            }
+        }
+        catch (e) {
+            if (mode === "vector")
+                throw e;
+        }
+    }
+    let hits;
+    if (mode === "vector") {
+        hits = vectorHits.map((h) => rowToHit(h.row, h.score, "vector")).slice(0, limit);
+    }
+    else if (mode === "fts") {
+        const norm = normalizeFts(ftsHits.map((h) => h.bm25));
+        hits = ftsHits
+            .map((h) => rowToHit(h.row, norm.get(h.bm25) ?? 0, "fts"))
+            .slice(0, limit);
+    }
+    else {
+        // hybrid 合并
+        const merged = new Map();
+        for (const h of vectorHits) {
+            merged.set(h.row.id, rowToHit(h.row, h.score, "vector"));
+        }
+        const norm = normalizeFts(ftsHits.map((h) => h.bm25));
+        for (const h of ftsHits) {
+            const ftsScore = norm.get(h.bm25) ?? 0;
+            const existed = merged.get(h.row.id);
+            if (existed) {
+                merged.set(h.row.id, { ...existed, score: Math.max(existed.score, ftsScore), match: "both" });
+            }
+            else {
+                merged.set(h.row.id, rowToHit(h.row, ftsScore, "fts"));
+            }
+        }
+        hits = Array.from(merged.values())
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    return {
+        source: "aosp",
+        query: args.query,
+        mode,
+        hits,
+        filter: {
+            platform: args.platform,
+            module: args.module,
+            module_paths: modulePathPrefixes,
+        },
+    };
+}

package/dist/config.js

@@ -0,0 +1,39 @@
+/**
+ * 全局配置读取（从环境变量）。
+ *
+ * 路径与限制集中放这里，避免散落在各模块。
+ */
+import * as path from "node:path";
+import * as os from "node:os";
+function envInt(key, fallback) {
+    const raw = (process.env[key] ?? "").trim();
+    if (!/^\d+$/.test(raw))
+        return fallback;
+    const n = parseInt(raw, 10);
+    return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+export const config = {
+    /** SQLite 主库路径 */
+    dbPath: (process.env.KNOWLEDGE_DB_PATH ?? path.resolve("./data/knowledge.db")).trim(),
+    /** ONNX 嵌入模型缓存目录 */
+    modelCacheDir: (process.env.KNOWLEDGE_MODEL_CACHE_DIR ?? path.resolve("./data/models")).trim(),
+    /** 嵌入模型 ID（默认 BGE-small-zh ONNX，dim=512） */
+    embeddingModelId: (process.env.KNOWLEDGE_EMBEDDING_MODEL ?? "Xenova/bge-small-zh-v1.5").trim(),
+    /** 嵌入向量维度（必须与模型一致；BGE-small-zh-v1.5 是 512） */
+    embeddingDim: envInt("KNOWLEDGE_EMBEDDING_DIM", 512),
+    /** 嵌入 batch 大小 */
+    embeddingBatchSize: envInt("KNOWLEDGE_EMBEDDING_BATCH", 32),
+    /** ONNX runtime 线程数（防止低配机吃光） */
+    embeddingThreads: envInt("KNOWLEDGE_EMBEDDING_THREADS", Math.max(1, Math.min(4, Math.floor((os.cpus()?.length ?? 4) / 2)))),
+    /** 单条文本嵌入前的最大字符长度（防 token 超 BGE 512 上限） */
+    maxTextChars: envInt("KNOWLEDGE_MAX_TEXT_CHARS", 1800),
+    /** search_local 的默认 limit */
+    searchDefaultLimit: 5,
+    /** search_local 的 limit 上限 */
+    searchMaxLimit: envInt("KNOWLEDGE_SEARCH_MAX_LIMIT", 20),
+    /** snippet 截断字符数 */
+    snippetMaxChars: envInt("KNOWLEDGE_SNIPPET_MAX_CHARS", 320),
+};
+export function isSourceName(s) {
+    return s === "zmind" || s === "gerrit" || s === "confluence";
+}