dsh-shared-kb 0.0.0-stage → 1.0.0

package/lib/indexer/sqlite-index.js

@@ -0,0 +1,431 @@
+/**
+ * 本机索引库（SQLite / node:sqlite）。
+ *
+ * 关键决策：索引**不进共享盘**。
+ *
+ *   共享盘（SMB / UNC / NAS）上放一个多进程同时写的 SQLite 文件是灾难：
+ *   SQLite 的 WAL 依赖共享内存映射与 fcntl 锁，SMB 实现普遍不支持或
+ *   语义不一致，典型后果是索引静默损坏、无法打开、或整库被标记为
+ *   malformed 之后每次打开都要重建。
+ *
+ *   所以：**资料在共享盘（唯一真相），索引在本机**（每台机器一份）。
+ *   每台机器按根目录路径哈希出自己的缓存目录：<home>/indexes/<hash>/kb-<name>.sqlite
+ *   索引记录它对应的 manifest rev 与各文档 version；启动/检索时只比对
+ *   版本号做增量刷新，别人在别的机器上导了资料，我这里第一次检索就把
+ *   那几篇补进来，不需要重建全库。
+ *
+ * 表结构：
+ *   parts(doc_id, part, title, heading, text, text_indexed, page, chars, tokens, vec)
+ *   parts_fts  ← FTS5 虚表（对 text_indexed 建索引）
+ *   docs(doc_id, version, title, updated_at, chars)
+ *   meta(key, value)  ← provider 名/维度、schema 版本、构建时间
+ *   vecs(doc_id, part, v BLOB)  ← 向量单独存，避免 FTS 表被大 BLOB 拖慢
+ */
+import { mkdir, readFile, rename, rm, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { HubError } from "../core/errors.js";
+import { indexText } from "./tokenize.js";
+
+export const INDEX_SCHEMA = 3;
+
+let sqliteModulePromise = null;
+
+/** 懒加载 node:sqlite；不可用时抛可诊断错误（不静默降级成「没有检索」）。 */
+export async function loadSqlite() {
+  if (!sqliteModulePromise) {
+    sqliteModulePromise = import("node:sqlite").then(
+      (mod) => mod,
+      (error) => {
+        throw new HubError(
+          "SQLITE_UNAVAILABLE",
+          `当前 Node 运行时没有 node:sqlite（需要 >=22.5，且需 --experimental-sqlite 或已稳定内置）：${String(error?.message ?? error)}`,
+          { status: 500 }
+        );
+      }
+    );
+  }
+  return sqliteModulePromise;
+}
+
+export function indexPathFor(indexRoot, kbName) {
+  return join(indexRoot, `kb-${kbName}.sqlite`);
+}
+
+/**
+ * 打开（或创建）某个库的本机索引。
+ * @param {string} path sqlite 文件路径
+ * @param {{provider: string, dim: number}} identity 嵌入身份；与库内记录不一致就整体重建
+ */
+export async function openIndex(path, identity) {
+  const { DatabaseSync } = await loadSqlite();
+  await mkdir(join(path, ".."), { recursive: true });
+  let db;
+  try {
+    db = new DatabaseSync(path);
+  } catch (error) {
+    // 文件损坏：删掉重建，而不是让用户卡在「检索一直报错」
+    await rm(path, { force: true }).catch(() => {});
+    db = new DatabaseSync(path);
+  }
+  db.exec("pragma journal_mode=WAL");
+  db.exec("pragma synchronous=NORMAL");
+  db.exec("pragma busy_timeout=8000");
+  db.exec("pragma temp_store=MEMORY");
+
+  ensureSchema(db);
+
+  const stored = readMeta(db);
+  const mismatch =
+    stored.schema !== INDEX_SCHEMA ||
+    stored.provider !== identity.provider ||
+    Number(stored.dim) !== Number(identity.dim);
+  if (mismatch) {
+    wipe(db);
+    writeMeta(db, {
+      schema: INDEX_SCHEMA,
+      provider: identity.provider,
+      dim: String(identity.dim),
+      createdAt: new Date().toISOString(),
+    });
+  }
+
+  return new IndexHandle(db, identity);
+}
+
+function ensureSchema(db) {
+  db.exec(`
+    create table if not exists meta (key text primary key, value text not null);
+    create table if not exists docs (
+      doc_id text primary key,
+      version integer not null,
+      title text not null default '',
+      updated_at text not null default '',
+      chars integer not null default 0
+    );
+    create table if not exists parts (
+      doc_id text not null,
+      part integer not null,
+      title text not null default '',
+      heading text not null default '',
+      text text not null default '',
+      page integer,
+      chars integer not null default 0,
+      tokens integer not null default 0,
+      primary key (doc_id, part)
+    );
+    create table if not exists vecs (
+      doc_id text not null,
+      part integer not null,
+      v blob not null,
+      primary key (doc_id, part)
+    );
+    create virtual table if not exists parts_fts using fts5(
+      doc_id UNINDEXED,
+      part UNINDEXED,
+      body,
+      title,
+      tokenize='unicode61 remove_diacritics 2'
+    );
+    create index if not exists parts_doc on parts(doc_id);
+  `);
+}
+
+function wipe(db) {
+  db.exec("delete from parts_fts; delete from parts; delete from vecs; delete from docs; delete from meta;");
+}
+
+function readMeta(db) {
+  const out = {};
+  try {
+    for (const row of db.prepare("select key, value from meta").all()) out[row.key] = row.value;
+  } catch {
+    /* 表可能刚被删 */
+  }
+  return out;
+}
+
+function writeMeta(db, entries) {
+  const stmt = db.prepare("insert into meta(key, value) values(?, ?) on conflict(key) do update set value=excluded.value");
+  for (const [key, value] of Object.entries(entries)) stmt.run(key, String(value));
+}
+
+function readMetaValue(db, key) {
+  try {
+    const row = db.prepare("select value from meta where key = ?").get(key);
+    return row?.value ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/** 把 Float32Array 与 BLOB 互转。存原始字节，比 JSON 省 3-4 倍空间。 */
+function vectorToBuffer(vector) {
+  return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+function bufferToVector(buffer) {
+  const bytes = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+  return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
+}
+
+class IndexHandle {
+  constructor(db, identity) {
+    this.db = db;
+    this.identity = identity;
+    this.closed = false;
+  }
+
+  /** 索引里当前有哪些文档、各是哪一版（用于与 manifest 比对增量）。 */
+  indexedVersions() {
+    const out = new Map();
+    for (const row of this.db.prepare("select doc_id, version from docs").all()) {
+      out.set(row.doc_id, Number(row.version));
+    }
+    return out;
+  }
+
+  getStamp() {
+    return readMetaValue(this.db, "manifestRev");
+  }
+
+  /**
+   * 用一批文档替换索引内容。
+   * @param {{docId: string, version: number, title: string, updatedAt: string, chars: number, parts: object[], vectors?: Float32Array[]}[]} docs
+   * @param {{manifestRev: number}} meta
+   */
+  upsertDocs(docs, meta = {}) {
+    const delFts = this.db.prepare("delete from parts_fts where doc_id = ?");
+    const delParts = this.db.prepare("delete from parts where doc_id = ?");
+    const delVecs = this.db.prepare("delete from vecs where doc_id = ?");
+    const upsertDoc = this.db.prepare(
+      "insert into docs(doc_id, version, title, updated_at, chars) values(?,?,?,?,?) " +
+        "on conflict(doc_id) do update set version=excluded.version, title=excluded.title, updated_at=excluded.updated_at, chars=excluded.chars"
+    );
+    const insPart = this.db.prepare(
+      "insert into parts(doc_id, part, title, heading, text, page, chars, tokens) values(?,?,?,?,?,?,?,?)"
+    );
+    const insFts = this.db.prepare("insert into parts_fts(doc_id, part, body, title) values(?,?,?,?)");
+    const insVec = this.db.prepare("insert into vecs(doc_id, part, v) values(?,?,?)");
+
+    this.db.exec("begin immediate");
+    try {
+      for (const doc of docs) {
+        delFts.run(doc.docId);
+        delParts.run(doc.docId);
+        delVecs.run(doc.docId);
+        upsertDoc.run(doc.docId, Number(doc.version) || 1, String(doc.title ?? ""), String(doc.updatedAt ?? ""), Number(doc.chars) || 0);
+        const parts = Array.isArray(doc.parts) ? doc.parts : [];
+        for (const part of parts) {
+          const index = Number(part.index) || 0;
+          insPart.run(
+            doc.docId,
+            index,
+            String(doc.title ?? ""),
+            String(part.heading ?? ""),
+            String(part.text ?? ""),
+            part.page == null ? null : Number(part.page),
+            Number(part.chars) || String(part.text ?? "").length,
+            Number(part.tokens) || 0
+          );
+          insFts.run(doc.docId, index, indexText(part.text ?? ""), indexText(`${doc.title ?? ""} ${part.heading ?? ""}`));
+        }
+        const vectors = Array.isArray(doc.vectors) ? doc.vectors : [];
+        for (let i = 0; i < parts.length && i < vectors.length; i += 1) {
+          if (vectors[i]) insVec.run(doc.docId, Number(parts[i].index) || i + 1, vectorToBuffer(vectors[i]));
+        }
+      }
+      if (meta.manifestRev !== undefined) {
+        this.db.prepare("insert into meta(key, value) values('manifestRev', ?) on conflict(key) do update set value=excluded.value").run(
+          String(meta.manifestRev)
+        );
+      }
+      if (meta.builtAt) {
+        this.db
+          .prepare("insert into meta(key, value) values('builtAt', ?) on conflict(key) do update set value=excluded.value")
+          .run(String(meta.builtAt));
+      }
+      this.db.exec("commit");
+    } catch (error) {
+      this.db.exec("rollback");
+      throw error;
+    }
+  }
+
+  /** 从索引移除若干文档（软删/彻底删）。 */
+  removeDocs(docIds) {
+    const delFts = this.db.prepare("delete from parts_fts where doc_id = ?");
+    const delParts = this.db.prepare("delete from parts where doc_id = ?");
+    const delVecs = this.db.prepare("delete from vecs where doc_id = ?");
+    const delDoc = this.db.prepare("delete from docs where doc_id = ?");
+    this.db.exec("begin immediate");
+    try {
+      for (const docId of docIds) {
+        delFts.run(docId);
+        delParts.run(docId);
+        delVecs.run(docId);
+        delDoc.run(docId);
+      }
+      this.db.exec("commit");
+    } catch (error) {
+      this.db.exec("rollback");
+      throw error;
+    }
+  }
+
+  docCount() {
+    try {
+      return Number(this.db.prepare("select count(*) c from docs").get()?.c ?? 0);
+    } catch {
+      return 0;
+    }
+  }
+
+  partCount() {
+    try {
+      return Number(this.db.prepare("select count(*) c from parts").get()?.c ?? 0);
+    } catch {
+      return 0;
+    }
+  }
+
+  /**
+   * 全文检索（BM25）。
+   * @param {string} matchExpr FTS5 MATCH 表达式
+   * @param {{limit?: number, docIds?: string[]}} options
+   */
+  searchFullText(matchExpr, options = {}) {
+    const limit = Math.max(1, Math.min(200, Math.round(Number(options.limit) || 40)));
+    if (!matchExpr) return [];
+    const allow = Array.isArray(options.docIds) && options.docIds.length > 0 ? new Set(options.docIds) : null;
+    const sql =
+      "select parts_fts.doc_id as doc_id, parts_fts.part as part, bm25(parts_fts) as score, " +
+      "parts.text as text, parts.title as title, parts.heading as heading, parts.page as page " +
+      "from parts_fts join parts on parts.doc_id = parts_fts.doc_id and parts.part = parts_fts.part " +
+      "where parts_fts match ? order by score limit ?";
+    let rows;
+    try {
+      rows = this.db.prepare(sql).all(matchExpr, allow ? limit * 4 : limit);
+    } catch (error) {
+      // 非法 MATCH 表达式（用户输入带引号/特殊字符）不能让检索整体失败
+      throw new HubError("QUERY_SYNTAX", `检索表达式无法解析：${String(error?.message ?? error)}`, { status: 400 });
+    }
+    const out = [];
+    for (const row of rows) {
+      if (allow && !allow.has(row.doc_id)) continue;
+      out.push({
+        docId: String(row.doc_id),
+        part: Number(row.part),
+        // bm25 越小越相关；取负号变成「越大越相关」，方便与向量分数融合
+        score: -Number(row.score),
+        text: String(row.text ?? ""),
+        title: String(row.title ?? ""),
+        heading: String(row.heading ?? ""),
+        page: row.page == null ? null : Number(row.page),
+      });
+      if (out.length >= limit) break;
+    }
+    return out;
+  }
+
+  /**
+   * 向量检索（全表扫描 + 点积）。
+   *
+   * 为什么不用 sqlite-vec：额外原生扩展会带来安装与跨平台负担，
+   * 而知识库规模（几千到几万块）下，Float32 全扫一遍在内存里是毫秒级。
+   * 超过阈值（默认 20 万块）时上层会提示「建议开启分库」而不是硬扛。
+   */
+  searchVector(queryVector, options = {}) {
+    const limit = Math.max(1, Math.min(200, Math.round(Number(options.limit) || 40)));
+    const allow = Array.isArray(options.docIds) && options.docIds.length > 0 ? new Set(options.docIds) : null;
+    const rows = this.db.prepare("select v.doc_id as doc_id, v.part as part, v.v as v from vecs v").all();
+    const scored = [];
+    const q = queryVector;
+    for (const row of rows) {
+      if (allow && !allow.has(row.doc_id)) continue;
+      const vector = bufferToVector(row.v);
+      if (vector.length !== q.length) continue;
+      let sum = 0;
+      for (let i = 0; i < vector.length; i += 1) sum += vector[i] * q[i];
+      scored.push({ docId: String(row.doc_id), part: Number(row.part), score: sum });
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const top = scored.slice(0, limit);
+    const detail = new Map();
+    if (top.length > 0) {
+      const stmt = this.db.prepare(
+        "select text, title, heading, page from parts where doc_id = ? and part = ?"
+      );
+      for (const hit of top) {
+        const row = stmt.get(hit.docId, hit.part);
+        if (row) {
+          detail.set(`${hit.docId}#${hit.part}`, {
+            text: String(row.text ?? ""),
+            title: String(row.title ?? ""),
+            heading: String(row.heading ?? ""),
+            page: row.page == null ? null : Number(row.page),
+          });
+        }
+      }
+    }
+    return top.map((hit) => ({
+      ...hit,
+      ...(detail.get(`${hit.docId}#${hit.part}`) ?? { text: "", title: "", heading: "", page: null }),
+    }));
+  }
+
+  /** 取某篇文档的全文块（供精读）。 */
+  readParts(docId, { from = 1, to = Infinity } = {}) {
+    const rows = this.db
+      .prepare("select part, text, heading, page, chars from parts where doc_id = ? and part >= ? and part <= ? order by part")
+      .all(docId, from, to === Infinity ? 1e9 : to);
+    return rows.map((row) => ({
+      index: Number(row.part),
+      text: String(row.text ?? ""),
+      heading: String(row.heading ?? ""),
+      page: row.page == null ? null : Number(row.page),
+      chars: Number(row.chars) || 0,
+    }));
+  }
+
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    try {
+      this.db.close();
+    } catch {
+      /* 已关闭 */
+    }
+  }
+}
+
+/** 索引健康信息（给状态页与 kb_index_status 用）。 */
+export async function indexFileInfo(path) {
+  try {
+    const s = await stat(path);
+    return { exists: true, size: s.size, mtime: new Date(s.mtimeMs).toISOString() };
+  } catch {
+    return { exists: false, size: 0, mtime: null };
+  }
+}
+
+/** 索引文件迁移辅助：把一个索引文件挪到备份名（重建前留底）。 */
+export async function moveIndexAside(path) {
+  const backup = `${path}.bak-${Date.now()}`;
+  try {
+    await rename(path, backup);
+    return backup;
+  } catch {
+    return null;
+  }
+}
+
+/** 从磁盘读一个 .f32 文件（兼容旧格式，供将来导入用）。 */
+export async function readVectorFile(path, dim) {
+  const buffer = await readFile(path);
+  const count = Math.floor(buffer.length / (dim * 4));
+  const out = [];
+  for (let i = 0; i < count; i += 1) {
+    const offset = i * dim * 4;
+    out.push(new Float32Array(buffer.buffer, buffer.byteOffset + offset, dim));
+  }
+  return out;
+}

package/lib/indexer/tokenize.js

@@ -0,0 +1,151 @@
+/**
+ * 中文全文索引的分词策略。
+ *
+ * SQLite FTS5 的 unicode61 分词器不认识中文：它会把整段连续汉字当成一个词，
+ * 于是「知识库检索」在索引里是一个 token，用户搜「知识库」命中不了。
+ *
+ * 两条路：
+ *  a) 用 trigram tokenizer（FTS5 内置）：任意 3 字符子串都能命中，中文效果好，
+ *     但索引体积约为原文 3-4 倍，且不支持前缀/短语查询的良好语义；
+ *  b) 自己预分词后存进普通 unicode61 列：索引小、可控、能做精确短语匹配。
+ *
+ * 这里选 b，理由：知识库常有标准号、型号、条款号这类必须精确匹配的内容，
+ * trigram 会把 "GB/T 19963.2-2024" 切得像噪音；预分词可以同时保留
+ * 中文二元组 + 英文词 + 数字串，并且让「标准号不同写法」有统一入口。
+ *
+ * 分词规则（输出用空格连接的 token 串，供 FTS5 索引）：
+ *   1. Intl.Segmenter 中文分词（Node 内置，无需第三方词典）
+ *   2. 汉字连续段额外产出二元组（bigram）：解决分词器把「知识库」切成
+ *      「知识」「库」后搜「识库」命不中的问题
+ *   3. 英文/数字：整词 + 去分隔符压紧形式（GB/T → gbt）
+ *   4. 数字串数字面量（含小数、区间），保证「19963.2」可查
+ *
+ * 检索侧复用同一函数：索引与查询用同一套分词，才不会「索引里有、查不到」。
+ */
+
+let segmenter = null;
+function getSegmenter() {
+  if (segmenter === null) {
+    try {
+      segmenter = new Intl.Segmenter("zh-Hans", { granularity: "word" });
+    } catch {
+      segmenter = false;
+    }
+  }
+  return segmenter || null;
+}
+
+const CJK = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
+const CJK_RUN = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/g;
+
+/** 归一化：全角转半角、大小写统一、去零宽字符。 */
+export function normalizeForIndex(input) {
+  return String(input ?? "")
+    .normalize("NFKC")
+    .replace(/[\u200b-\u200f\u2060\ufeff]/g, "")
+    .toLowerCase();
+}
+
+/**
+ * 把文本切成用于索引/查询的 token 列表。
+ * @param {string} input
+ * @param {{bigram?: boolean}} [options] bigram 默认 true（索引侧开、查询侧也开）
+ */
+export function tokenize(input, options = {}) {
+  const text = normalizeForIndex(input);
+  const tokens = new Set();
+  if (text.trim() === "") return [];
+
+  const bigram = options.bigram !== false;
+
+  // 1. Segmenter 词级
+  const seg = getSegmenter();
+  if (seg) {
+    for (const item of seg.segment(text)) {
+      const word = String(item.segment ?? "").trim();
+      if (word === "") continue;
+      if (word.length === 1 && !CJK.test(word)) continue;
+      if (/^[\p{L}\p{N}]+$/u.test(word)) tokens.add(word);
+    }
+  }
+
+  // 2. 汉字二元组
+  if (bigram) {
+    for (const run of text.match(CJK_RUN) ?? []) {
+      if (run.length === 1) {
+        tokens.add(run);
+        continue;
+      }
+      for (let i = 0; i + 1 < run.length; i += 1) tokens.add(run.slice(i, i + 2));
+      // 三字以上再补首尾二字组合，覆盖「检索增强」类复合词
+      if (run.length >= 4) {
+        tokens.add(run.slice(0, 2));
+        tokens.add(run.slice(-2));
+      }
+    }
+  }
+
+  // 3. 英文词 / 数字 / 型号 / 标准号
+  for (const word of text.match(/[a-z][a-z0-9_+#.-]{0,40}/g) ?? []) {
+    if (word.length < 2) continue;
+    tokens.add(word);
+    const compact = word.replace(/[^a-z0-9]/g, "");
+    if (compact.length >= 2 && compact !== word) tokens.add(compact);
+  }
+  for (const num of text.match(/\d+(?:[.,]\d+)*(?:[-–~]\d+(?:[.,]\d+)*)?/g) ?? []) {
+    tokens.add(num);
+    const stripped = num.replace(/[.,]$/, "");
+    if (stripped !== num) tokens.add(stripped);
+  }
+
+  return [...tokens];
+}
+
+/** 索引侧：token 串（空格分隔），直接存进 FTS5 的文本列。 */
+export function indexText(input) {
+  return tokenize(input).join(" ");
+}
+
+/**
+ * 查询侧：把用户输入转成 FTS5 MATCH 表达式。
+ *
+ * 逐 token 用 OR 连接，整串再用引号包住避免语法注入；
+ * 同时给你「短语优先」的变体（相邻 token 用 AND），供上层做二级排序。
+ */
+export function buildMatchQuery(input, { mode = "or" } = {}) {
+  const tokens = tokenize(input);
+  if (tokens.length === 0) return null;
+  const quoted = tokens.map((token) => `"${token.replace(/"/g, '""')}"`);
+  if (mode === "and") return quoted.join(" AND ");
+  if (mode === "phrase") return quoted.join(" ");
+  return quoted.join(" OR ");
+}
+
+/**
+ * 判断查询是否只含「无意义 token」（如纯标点），避免生成空 MATCH 报错。
+ */
+export function hasQueryTokens(input) {
+  return tokenize(input).length > 0;
+}
+
+/**
+ * 把原文里命中的 token 高亮出来（返回片段），用于检索结果展示。
+ * 不用 FTS5 的 snippet()，因为索引列存的是分词后的串，不是原文。
+ */
+export function makeSnippet(text, query, { before = 40, after = 60, maxLen = 220 } = {}) {
+  const source = String(text ?? "").replace(/\s+/g, " ").trim();
+  if (source === "") return "";
+  const tokens = tokenize(query, { bigram: false });
+  const lower = normalizeForIndex(source);
+  let hit = -1;
+  for (const token of tokens) {
+    const at = lower.indexOf(token);
+    if (at >= 0 && (hit < 0 || at < hit)) hit = at;
+  }
+  if (hit < 0) return source.length > maxLen ? source.slice(0, maxLen) + "…" : source;
+  const start = Math.max(0, hit - before);
+  const end = Math.min(source.length, hit + after);
+  const prefix = start > 0 ? "…" : "";
+  const suffix = end < source.length ? "…" : "";
+  return prefix + source.slice(start, end) + suffix;
+}

package/lib/locale/zh-CN.json

@@ -0,0 +1,4 @@
+{
+  "name": "知识库中枢",
+  "description": "指定物理地址，每库独立子文件夹；FTS5 全文 + 向量语义检索；多人多机并发安全"
+}