@puwenhui/dsh-rag-kb 0.2.0 → 0.3.0

package/README.md

@@ -50,7 +50,7 @@ dsh plugin --profile web add @puwenhui/dsh-rag-kb
 ## 检索架构
 
 ```
-文档 → 解析(pdf/docx/md) → 标题感知切块(400 token + 10% 重叠)
+文档 → 解析(pdf/docx/md) → 标题感知切块(节标题强制断块 + 400 token + 10% 重叠)
   → 向量化(transformers.js bge-small-zh-v1.5, 512维, GPU/CPU)
   → FTS5 trigram 关键词索引（与向量索引同步写入）
 
@@ -59,6 +59,20 @@ dsh plugin --profile web add @puwenhui/dsh-rag-kb
   → 引用溯源(citation + position 字段)
 ```
 
+### 分块策略（v0.3.0 起）
+
+面向 Markdown 手册/知识库文档优化，保证**一节 = 一块**，引用溯源可精确到节：
+
+| 规则 | 说明 |
+|---|---|
+| 节标题强制断块 | 遇 `##` ~ `######` 立即断块，短节不再被合并；刻意不含单级 `#`——配置/代码注释普遍用它开头，误判会把命令序列切碎 |
+| 续块补回节标题 | 长节切成多块时，每块都带节标题，检索命中时能看清出处 |
+| 软下限 + 硬上限 | 块过短（如仅「节标题 + 来源行」）宁可略微超长也不单独成块；合并超过 1.2 倍上限则强制断开 |
+| 滑窗按换行对齐 | 超长无结构段落按换行位置切分，不把命令行从中间截断 |
+| 兜底宽度 | 块长控制在约 600 字符（≈400 token），实测上限 ~920 字符，避免超出 embedding 序列长度被截断 |
+
+实测效果（13 篇技术/管理文档，114 节）：切成 605 块，**96% 的块带节标题**。
+
 ## 技术栈
 
 | 层 | 选型 | 说明 |

package/lib/plugin.mjs

@@ -266,7 +266,12 @@ var indexer_exports = /* @__PURE__ */ __exportAll({
 });
 /** 中文友好的切块参数 */
 const CHUNK_TOKENS = 400;
-const OVERLAP_RATIO = .1;
+/**
+* 节标题行：`##` ~ `######` 视为节标题，遇之强制断块（保证「一节 = 一块」）。
+* 刻意不含单级 `#`——nginx/shell/mysql 等配置与代码的注释普遍用单 `#` 开头，
+* 误判会把命令序列切碎（实测语音平台文档 27 节被切成 231 块）。
+*/
+const HEADING_RE = /^#{2,6} /;
 async function extractText(filePath) {
 	const ext = extname(filePath).toLowerCase();
 	const buf = await readFile(filePath);
@@ -285,28 +290,47 @@ async function extractText(filePath) {
 	if (ext === ".md" || ext === ".txt" || ext === ".log" || ext === ".csv" || ext === ".json") return { text: buf.toString("utf8") };
 	throw new Error(`不支持的文件类型 ${ext}（支持 pdf/docx/md/txt/csv/json）`);
 }
-/** 标题感知切块：按标题/空行分段，过长段再固定长度切 */
+/** 标题感知切块：节标题强制断块（一节 = 一块）+ 空行分段 + 过长滑窗兜底 */
 function chunkText(text) {
 	const normalized = text.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
 	if (normalized === "") return [];
-	const paragraphs = normalized.split(/\n(?=#{1,4} )|\n\n+/).map((p) => p.trim()).filter((p) => p !== "");
+	const paragraphs = normalized.split(/\n(?=#{2,6} )|\n\n+/).map((p) => p.trim()).filter((p) => p !== "");
 	const chunks = [];
 	let current = "";
+	let sectionHeading = "";
 	const maxChars = CHUNK_TOKENS * 1.5;
-	const overlap = Math.floor(maxChars * OVERLAP_RATIO);
 	for (const para of paragraphs) {
-		if (current !== "" && (current + "\n" + para).length > maxChars) {
+		const isHeading = HEADING_RE.test(para);
+		if (isHeading) sectionHeading = para.split("\n")[0].trim();
+		if (isHeading && current !== "") {
 			chunks.push(current);
-			current = current.length > overlap ? current.slice(-60) : "";
+			current = "";
 		}
-		if (para.length > maxChars * 2) {
-			if (current !== "") {
-				chunks.push(current);
-				current = "";
+		if (para.length > maxChars * 1.2) {
+			const prefix = current;
+			current = "";
+			const mergePrefix = prefix !== "" && prefix.length <= maxChars * .6;
+			if (prefix !== "" && !mergePrefix) chunks.push(prefix);
+			for (let i = 0; i < para.length;) {
+				let end = Math.min(i + maxChars, para.length);
+				if (end < para.length) {
+					const nl = para.lastIndexOf("\n", end);
+					if (nl > i + maxChars * .5) end = nl + 1;
+				}
+				const piece = para.slice(i, end);
+				const isFirst = i === 0;
+				i = end;
+				chunks.push(isFirst ? mergePrefix ? `${prefix}\n\n${piece}` : piece : sectionHeading !== "" ? `${sectionHeading}\n\n${piece}` : piece);
 			}
-			for (let i = 0; i < para.length; i += 540) chunks.push(para.slice(i, i + maxChars));
+			current = sectionHeading;
 			continue;
 		}
+		const merged = current + "\n" + para;
+		if (current !== "" && merged.length > maxChars && (current.length >= maxChars * .5 || merged.length > maxChars * 1.2)) {
+			chunks.push(current);
+			const tail = current.slice(-60);
+			current = sectionHeading !== "" ? `${sectionHeading}\n\n${tail}` : tail;
+		}
 		current = current === "" ? para : current + "\n" + para;
 	}
 	if (current !== "") chunks.push(current);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@puwenhui/dsh-rag-kb",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "DSH RAG 知识库插件：混合检索(向量+FTS5)+Reranker 重排+GPU 加速+引用溯源+会话语义管理",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",