@kk-irving/knowledge-mcp-server 1.0.0

package/dist/db.js

@@ -0,0 +1,244 @@
+/**
+ * SQLite 数据层（基于 Node.js 22.5+ 内置 `node:sqlite`，无 native 编译）。
+ *
+ * 三源主表 + 各自 FTS5 + sync_state，与 v1.0 设计一致。
+ */
+import { DatabaseSync } from "node:sqlite";
+import * as path from "node:path";
+import { mkdirSync } from "node:fs";
+import { config } from "./config.js";
+let _db = null;
+export function getDb() {
+    if (_db)
+        return _db;
+    mkdirSync(path.dirname(config.dbPath), { recursive: true });
+    const db = new DatabaseSync(config.dbPath);
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA synchronous = NORMAL");
+    db.exec("PRAGMA foreign_keys = ON");
+    migrate(db);
+    _db = db;
+    return db;
+}
+export function closeDb() {
+    if (_db) {
+        try {
+            _db.close();
+        }
+        catch {
+            /* ignore */
+        }
+        _db = null;
+    }
+}
+// =============================================================================
+// 迁移（幂等）
+// =============================================================================
+function migrate(db) {
+    db.exec(`
+    -- 公共：同步水位
+    CREATE TABLE IF NOT EXISTS sync_state (
+      source TEXT NOT NULL,
+      key    TEXT NOT NULL,
+      value  TEXT NOT NULL,
+      PRIMARY KEY (source, key)
+    );
+
+    -- ============ Zmind issues ============
+    CREATE TABLE IF NOT EXISTS zmind_issues (
+      id                  INTEGER PRIMARY KEY,
+      tracker             TEXT,
+      subject             TEXT,
+      description         TEXT,
+      status              TEXT,
+      assigned_to         TEXT,
+      project_id          INTEGER,
+      project_name        TEXT,
+      created_on          TEXT,
+      updated_on          TEXT,
+      embedding           BLOB,
+      embedding_updated_at TEXT
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS zmind_issues_fts USING fts5(
+      subject, description,
+      content='zmind_issues', content_rowid='id',
+      tokenize='unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS zmind_issues_ai AFTER INSERT ON zmind_issues BEGIN
+      INSERT INTO zmind_issues_fts(rowid, subject, description)
+      VALUES (new.id, new.subject, new.description);
+    END;
+    CREATE TRIGGER IF NOT EXISTS zmind_issues_ad AFTER DELETE ON zmind_issues BEGIN
+      INSERT INTO zmind_issues_fts(zmind_issues_fts, rowid, subject, description)
+      VALUES('delete', old.id, old.subject, old.description);
+    END;
+    CREATE TRIGGER IF NOT EXISTS zmind_issues_au AFTER UPDATE ON zmind_issues BEGIN
+      INSERT INTO zmind_issues_fts(zmind_issues_fts, rowid, subject, description)
+      VALUES('delete', old.id, old.subject, old.description);
+      INSERT INTO zmind_issues_fts(rowid, subject, description)
+      VALUES (new.id, new.subject, new.description);
+    END;
+
+    -- ============ Gerrit changes ============
+    CREATE TABLE IF NOT EXISTS gerrit_changes (
+      change_id           TEXT PRIMARY KEY,
+      number              INTEGER,
+      project             TEXT,
+      branch              TEXT,
+      subject             TEXT,
+      commit_message      TEXT,
+      owner_name          TEXT,
+      status              TEXT,
+      created             TEXT,
+      updated             TEXT,
+      embedding           BLOB,
+      embedding_updated_at TEXT
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS gerrit_changes_fts USING fts5(
+      subject, commit_message,
+      content='gerrit_changes', content_rowid='rowid',
+      tokenize='unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS gerrit_changes_ai AFTER INSERT ON gerrit_changes BEGIN
+      INSERT INTO gerrit_changes_fts(rowid, subject, commit_message)
+      VALUES (new.rowid, new.subject, new.commit_message);
+    END;
+    CREATE TRIGGER IF NOT EXISTS gerrit_changes_ad AFTER DELETE ON gerrit_changes BEGIN
+      INSERT INTO gerrit_changes_fts(gerrit_changes_fts, rowid, subject, commit_message)
+      VALUES('delete', old.rowid, old.subject, old.commit_message);
+    END;
+    CREATE TRIGGER IF NOT EXISTS gerrit_changes_au AFTER UPDATE ON gerrit_changes BEGIN
+      INSERT INTO gerrit_changes_fts(gerrit_changes_fts, rowid, subject, commit_message)
+      VALUES('delete', old.rowid, old.subject, old.commit_message);
+      INSERT INTO gerrit_changes_fts(rowid, subject, commit_message)
+      VALUES (new.rowid, new.subject, new.commit_message);
+    END;
+
+    -- ============ Confluence pages ============
+    CREATE TABLE IF NOT EXISTS confluence_pages (
+      id                  TEXT PRIMARY KEY,
+      space_key           TEXT,
+      title               TEXT,
+      body_text           TEXT,
+      version             INTEGER,
+      webui               TEXT,
+      created             TEXT,
+      updated             TEXT,
+      embedding           BLOB,
+      embedding_updated_at TEXT
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS confluence_pages_fts USING fts5(
+      title, body_text,
+      content='confluence_pages', content_rowid='rowid',
+      tokenize='unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS confluence_pages_ai AFTER INSERT ON confluence_pages BEGIN
+      INSERT INTO confluence_pages_fts(rowid, title, body_text)
+      VALUES (new.rowid, new.title, new.body_text);
+    END;
+    CREATE TRIGGER IF NOT EXISTS confluence_pages_ad AFTER DELETE ON confluence_pages BEGIN
+      INSERT INTO confluence_pages_fts(confluence_pages_fts, rowid, title, body_text)
+      VALUES('delete', old.rowid, old.title, old.body_text);
+    END;
+    CREATE TRIGGER IF NOT EXISTS confluence_pages_au AFTER UPDATE ON confluence_pages BEGIN
+      INSERT INTO confluence_pages_fts(confluence_pages_fts, rowid, title, body_text)
+      VALUES('delete', old.rowid, old.title, old.body_text);
+      INSERT INTO confluence_pages_fts(rowid, title, body_text)
+      VALUES (new.rowid, new.title, new.body_text);
+    END;
+
+    -- ============ AOSP 代码 chunks (P2) ============
+    CREATE TABLE IF NOT EXISTS aosp_chunks (
+      id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+      platform            TEXT NOT NULL,
+      module              TEXT NOT NULL,
+      module_path         TEXT NOT NULL,
+      file_path           TEXT NOT NULL,
+      line_start          INTEGER NOT NULL,
+      line_end            INTEGER NOT NULL,
+      symbol_kind         TEXT,
+      symbol_name         TEXT,
+      content             TEXT NOT NULL,
+      content_hash        TEXT,
+      embedding           BLOB,
+      embedding_updated_at TEXT,
+      indexed_at          TEXT NOT NULL DEFAULT (datetime('now')),
+      UNIQUE(platform, module, file_path, line_start, line_end)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_aosp_pm ON aosp_chunks(platform, module);
+    CREATE INDEX IF NOT EXISTS idx_aosp_pmp ON aosp_chunks(platform, module_path);
+    CREATE INDEX IF NOT EXISTS idx_aosp_file ON aosp_chunks(file_path);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS aosp_chunks_fts USING fts5(
+      content,
+      content='aosp_chunks', content_rowid='id',
+      tokenize='unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS aosp_chunks_ai AFTER INSERT ON aosp_chunks BEGIN
+      INSERT INTO aosp_chunks_fts(rowid, content) VALUES (new.id, new.content);
+    END;
+    CREATE TRIGGER IF NOT EXISTS aosp_chunks_ad AFTER DELETE ON aosp_chunks BEGIN
+      INSERT INTO aosp_chunks_fts(aosp_chunks_fts, rowid, content) VALUES('delete', old.id, old.content);
+    END;
+    CREATE TRIGGER IF NOT EXISTS aosp_chunks_au AFTER UPDATE ON aosp_chunks BEGIN
+      INSERT INTO aosp_chunks_fts(aosp_chunks_fts, rowid, content) VALUES('delete', old.id, old.content);
+      INSERT INTO aosp_chunks_fts(rowid, content) VALUES (new.id, new.content);
+    END;
+  `);
+}
+// =============================================================================
+// 事务封装（node:sqlite 没有 better-sqlite3 的 transaction() helper）
+// =============================================================================
+export function runInTransaction(db, fn) {
+    db.exec("BEGIN");
+    try {
+        const result = fn();
+        db.exec("COMMIT");
+        return result;
+    }
+    catch (e) {
+        try {
+            db.exec("ROLLBACK");
+        }
+        catch {
+            /* ignore */
+        }
+        throw e;
+    }
+}
+// =============================================================================
+// sync_state 帮手
+// =============================================================================
+export function getSyncState(source, key) {
+    const row = getDb()
+        .prepare("SELECT value FROM sync_state WHERE source = ? AND key = ?")
+        .get(source, key);
+    return row?.value ?? null;
+}
+export function setSyncState(source, key, value) {
+    getDb()
+        .prepare("INSERT INTO sync_state(source, key, value) VALUES (?, ?, ?) ON CONFLICT(source, key) DO UPDATE SET value = excluded.value")
+        .run(source, key, value);
+}
+// =============================================================================
+// Helpers：序列化 / 反序列化向量
+// =============================================================================
+export function vectorToBlob(vec) {
+    return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+export function blobToVector(blob, dim) {
+    const ab = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength);
+    const f32 = new Float32Array(ab);
+    if (f32.length !== dim) {
+        throw new Error(`embedding dim mismatch: expected ${dim}, got ${f32.length}`);
+    }
+    return f32;
+}

package/dist/embed-pending.js

@@ -0,0 +1,81 @@
+/**
+ * embed_pending锛氭壒閲忓祵鍏?鏈祵鍏ユ垨宸茶繃鏈?鐨勮銆?
+ *
+ * "杩囨湡"鍒ゅ畾锛歟mbedding_updated_at 涓?NULL 鎴?< 涓昏〃鐨?updated 鍒椼€?
+ */
+import { getDb, runInTransaction, vectorToBlob } from "./db.js";
+import { embedTexts } from "./embedder.js";
+import { invalidateIndex } from "./index-store.js";
+function selectPending(source, limit) {
+    const db = getDb();
+    if (source === "zmind") {
+        return db
+            .prepare(`SELECT id AS _key,
+                COALESCE(subject, '') || char(10) || COALESCE(description, '') AS text
+           FROM zmind_issues
+          WHERE embedding IS NULL
+             OR embedding_updated_at IS NULL
+             OR (updated_on IS NOT NULL AND embedding_updated_at < updated_on)
+          ORDER BY updated_on DESC
+          LIMIT ?`)
+            .all(limit);
+    }
+    if (source === "gerrit") {
+        return db
+            .prepare(`SELECT change_id AS _key,
+                COALESCE(subject, '') || char(10) || COALESCE(commit_message, '') AS text
+           FROM gerrit_changes
+          WHERE embedding IS NULL
+             OR embedding_updated_at IS NULL
+             OR (updated IS NOT NULL AND embedding_updated_at < updated)
+          ORDER BY updated DESC
+          LIMIT ?`)
+            .all(limit);
+    }
+    return db
+        .prepare(`SELECT id AS _key,
+              COALESCE(title, '') || char(10) || COALESCE(body_text, '') AS text
+         FROM confluence_pages
+        WHERE embedding IS NULL
+           OR embedding_updated_at IS NULL
+           OR (updated IS NOT NULL AND embedding_updated_at < updated)
+        ORDER BY updated DESC
+        LIMIT ?`)
+        .all(limit);
+}
+function updateBlobs(source, rows) {
+    const db = getDb();
+    const sql = (() => {
+        if (source === "zmind")
+            return "UPDATE zmind_issues SET embedding = ?, embedding_updated_at = ? WHERE id = ?";
+        if (source === "gerrit")
+            return "UPDATE gerrit_changes SET embedding = ?, embedding_updated_at = ? WHERE change_id = ?";
+        return "UPDATE confluence_pages SET embedding = ?, embedding_updated_at = ? WHERE id = ?";
+    })();
+    const stmt = db.prepare(sql);
+    runInTransaction(db, () => {
+        for (const it of rows)
+            stmt.run(it.blob, it.ts, it.key);
+    });
+}
+export async function embedPending(args) {
+    const limit = Math.max(1, Math.min(5000, args.batch_size ?? 200));
+    const pending = selectPending(args.source, limit);
+    if (pending.length === 0) {
+        return { source: args.source, embedded: 0, total_pending: 0 };
+    }
+    const texts = pending.map((r) => r.text);
+    const vectors = await embedTexts(texts);
+    if (vectors.length !== pending.length) {
+        throw new Error(`embedTexts 杩斿洖 ${vectors.length} 椤癸紝鏈熸湜 ${pending.length}`);
+    }
+    const ts = new Date().toISOString();
+    const writeRows = pending.map((r, i) => ({
+        key: r._key,
+        blob: vectorToBlob(vectors[i]),
+        ts,
+    }));
+    updateBlobs(args.source, writeRows);
+    invalidateIndex(args.source);
+    return { source: args.source, embedded: pending.length, total_pending: pending.length };
+}

package/dist/embedder.js

@@ -0,0 +1,76 @@
+/**
+ * 嵌入器：单例 ONNX 模型 + batch encode。
+ *
+ * 默认模型 BAAI/bge-small-zh-v1.5（中文友好，512 dim，~80MB）。
+ * 通过 @xenova/transformers 在 Node 端运行 ONNX，无需 GPU。
+ *
+ * 首次调用会下载并缓存模型到 KNOWLEDGE_MODEL_CACHE_DIR（默认 ./data/models）。
+ */
+import { config } from "./config.js";
+let _embedderPromise = null;
+async function buildEmbedder() {
+    // 控制 ONNX runtime 线程数（防止低配机吃光）
+    process.env.OMP_NUM_THREADS = String(config.embeddingThreads);
+    process.env.MKL_NUM_THREADS = String(config.embeddingThreads);
+    const transformers = (await import("@xenova/transformers"));
+    // 重定向模型缓存目录
+    if (transformers?.env) {
+        transformers.env.cacheDir = config.modelCacheDir;
+        // 默认开启的远程下载
+        transformers.env.allowRemoteModels = true;
+    }
+    const pipeline = transformers.pipeline;
+    const extractor = await pipeline("feature-extraction", config.embeddingModelId, {
+        quantized: true, // 用 quantized 模型省内存
+    });
+    return async (texts) => {
+        if (texts.length === 0)
+            return [];
+        // 截断防止 token 超 BGE 512 上限
+        const trimmed = texts.map((t) => typeof t === "string" ? t.slice(0, config.maxTextChars) : "");
+        const out = await extractor(trimmed, { pooling: "mean", normalize: true });
+        // out.data 是 Float32Array，dims = [N, dim]
+        const dims = out.dims;
+        const dim = dims[dims.length - 1];
+        const flat = out.data;
+        if (dim !== config.embeddingDim) {
+            throw new Error(`模型输出维度 ${dim} 与 config.embeddingDim ${config.embeddingDim} 不一致；请调整 KNOWLEDGE_EMBEDDING_DIM`);
+        }
+        const result = [];
+        for (let i = 0; i < trimmed.length; i++) {
+            const start = i * dim;
+            // 必须复制独立 buffer（否则共享底层 ArrayBuffer 会被下次 batch 覆盖）
+            result.push(new Float32Array(flat.buffer.slice(flat.byteOffset + start * 4, flat.byteOffset + (start + dim) * 4)));
+        }
+        return result;
+    };
+}
+export async function getEmbedder() {
+    if (!_embedderPromise)
+        _embedderPromise = buildEmbedder();
+    return _embedderPromise;
+}
+/**
+ * 批量嵌入 N 条文本，按 config.embeddingBatchSize 切片调用 ONNX。
+ */
+export async function embedTexts(texts) {
+    if (texts.length === 0)
+        return [];
+    const embed = await getEmbedder();
+    const result = [];
+    const batch = Math.max(1, config.embeddingBatchSize);
+    for (let i = 0; i < texts.length; i += batch) {
+        const slice = texts.slice(i, i + batch);
+        const out = await embed(slice);
+        for (const v of out)
+            result.push(v);
+    }
+    return result;
+}
+/**
+ * 单条嵌入便捷方法。
+ */
+export async function embedOne(text) {
+    const out = await embedTexts([text]);
+    return out[0];
+}

package/dist/index-store.js

@@ -0,0 +1,175 @@
+/**
+ * 进程内向量索引：lazy 加载 + invalidate。
+ *
+ * 每个 source（zmind / gerrit / confluence）维护一份：
+ *   - ids: 主键数组
+ *   - matrix: 单一连续 Float32Array（N × dim 行优先），便于一次循环算 cosine
+ *   - meta: 主键 → 行元信息映射
+ */
+import { config } from "./config.js";
+import { blobToVector, getDb } from "./db.js";
+const _cache = new Map();
+export function invalidateIndex(source) {
+    if (source)
+        _cache.delete(source);
+    else
+        _cache.clear();
+}
+/**
+ * 加载某个 source 的所有 embedded rows 到内存矩阵。
+ * 首次调用或 invalidate 后下一次调用都会全表 SELECT。
+ */
+export function loadIndex(source) {
+    const cached = _cache.get(source);
+    if (cached)
+        return cached;
+    const db = getDb();
+    const dim = config.embeddingDim;
+    const rows = selectRowsWithEmbedding(source, db);
+    const matrix = new Float32Array(rows.length * dim);
+    const ids = [];
+    const meta = new Map();
+    rows.forEach((r, i) => {
+        const vec = blobToVector(r.embedding, dim);
+        matrix.set(vec, i * dim);
+        ids.push(r._key);
+        meta.set(r._key, r._meta);
+    });
+    const idx = { ids, matrix, count: rows.length, meta };
+    _cache.set(source, idx);
+    return idx;
+}
+function selectRowsWithEmbedding(source, db) {
+    if (source === "zmind") {
+        const rs = db
+            .prepare(`SELECT id, subject, description, status, project_name, updated_on, embedding
+           FROM zmind_issues
+          WHERE embedding IS NOT NULL`)
+            .all();
+        return rs.map((r) => ({
+            _key: r.id,
+            _meta: {
+                source: "zmind",
+                id: r.id,
+                title: r.subject ?? "",
+                url: zmindUrl(r.id),
+                snippet: snippet(r.description ?? "", config.snippetMaxChars),
+                status: r.status ?? "",
+                project: r.project_name ?? "",
+                updated: r.updated_on ?? "",
+            },
+            embedding: r.embedding,
+        }));
+    }
+    if (source === "gerrit") {
+        const rs = db
+            .prepare(`SELECT change_id, number, project, subject, commit_message, status, updated, embedding
+           FROM gerrit_changes
+          WHERE embedding IS NOT NULL`)
+            .all();
+        return rs.map((r) => ({
+            _key: r.change_id,
+            _meta: {
+                source: "gerrit",
+                id: r.change_id,
+                title: r.subject ?? "",
+                url: gerritUrl(r.project, r.number),
+                snippet: snippet(r.commit_message ?? "", config.snippetMaxChars),
+                status: r.status ?? "",
+                project: r.project ?? "",
+                updated: r.updated ?? "",
+                extra: { number: r.number },
+            },
+            embedding: r.embedding,
+        }));
+    }
+    // confluence
+    const rs = db
+        .prepare(`SELECT id, space_key, title, body_text, version, webui, updated, embedding
+         FROM confluence_pages
+        WHERE embedding IS NOT NULL`)
+        .all();
+    return rs.map((r) => ({
+        _key: r.id,
+        _meta: {
+            source: "confluence",
+            id: r.id,
+            title: r.title ?? "",
+            url: confluenceUrl(r.webui),
+            snippet: snippet(r.body_text ?? "", config.snippetMaxChars),
+            space: r.space_key ?? "",
+            updated: r.updated ?? "",
+            extra: { version: r.version },
+        },
+        embedding: r.embedding,
+    }));
+}
+// =============================================================================
+// URL 帮手
+// =============================================================================
+function zmindUrl(id) {
+    const base = (process.env.ZMIND_URL ?? "https://zmind.whaletv.com").replace(/\/+$/, "");
+    return `${base}/issues/${id}`;
+}
+function gerritUrl(project, num) {
+    const base = (process.env.GERRIT_URL ?? "").replace(/\/+$/, "");
+    if (!base || !num)
+        return "";
+    return `${base}/c/${project}/+/${num}`;
+}
+function confluenceUrl(webui) {
+    if (!webui)
+        return "";
+    if (/^https?:\/\//i.test(webui))
+        return webui;
+    const base = (process.env.CONFLUENCE_BASE_URL ?? "").replace(/\/+$/, "");
+    return base ? base + webui : webui;
+}
+function snippet(text, maxChars) {
+    const clean = text.replace(/\s+/g, " ").trim();
+    if (clean.length <= maxChars)
+        return clean;
+    return clean.slice(0, maxChars).trimEnd() + "…";
+}
+/**
+ * 计算 query 向量与 source 索引的 top-K cosine（向量已 L2 normalize，dot=cosine）。
+ *
+ * @returns 按分数降序的 [{ id, score, meta }, ...]
+ */
+export function vectorTopK(source, query, topK, excludeIds) {
+    const idx = loadIndex(source);
+    if (idx.count === 0)
+        return [];
+    const dim = config.embeddingDim;
+    if (query.length !== dim) {
+        throw new Error(`query dim ${query.length} != embedding dim ${dim}`);
+    }
+    // 简单全表 dot product
+    const scores = new Float32Array(idx.count);
+    for (let i = 0; i < idx.count; i++) {
+        const offset = i * dim;
+        let s = 0;
+        for (let j = 0; j < dim; j++) {
+            s += idx.matrix[offset + j] * query[j];
+        }
+        scores[i] = s;
+    }
+    // 取 top-K
+    const indices = [];
+    for (let i = 0; i < idx.count; i++) {
+        if (excludeIds && excludeIds.has(idx.ids[i]))
+            continue;
+        indices.push(i);
+    }
+    indices.sort((a, b) => scores[b] - scores[a]);
+    const out = [];
+    for (let i = 0; i < Math.min(topK, indices.length); i++) {
+        const idxRow = indices[i];
+        const id = idx.ids[idxRow];
+        const meta = idx.meta.get(id);
+        if (!meta)
+            continue;
+        out.push({ id, score: scores[idxRow], meta });
+    }
+    return out;
+}