dsh-shared-kb 0.0.0-stage → 1.0.0

package/lib/parse/chunk.js

@@ -0,0 +1,202 @@
+/**
+ * 分块（chunking）。
+ *
+ * 目标不是「切得均匀」，而是「切出来的每一块能独立回答一个问题」：
+ *
+ *  - 先按结构切（标题 / 条款 / 表格行组 / 幻灯片 / 工作表），结构是天然的语义边界；
+ *  - 结构块过大再按段落 + 句子回退切，带重叠（overlap）保住跨句上下文；
+ *  - 过小的块与相邻块合并（避免「第 3 条」这种只有编号的碎片进索引）；
+ *  - 每块记录 part 序号、页码、标题路径（heading path），
+ *    这样检索命中后能给出「文档 > 第 2 章 > 3.1 节，第 12 页」的可复核定位。
+ */
+
+/** 约 1 token ≈ 1.6 个汉字；part 默认 1200 字符 ≈ 750 token。 */
+export const DEFAULT_PART_CHARS = 1200;
+export const DEFAULT_OVERLAP_CHARS = 150;
+export const MIN_PART_CHARS = 60;
+
+/** 中文按字符、英文按词估算 token（只用于控制块大小，不追求精确）。 */
+export function estimateTokens(text) {
+  const s = String(text ?? "");
+  const cjk = (s.match(/[\u3400-\u9fff]/g) ?? []).length;
+  const rest = s.length - cjk;
+  return Math.ceil(cjk / 1.6 + rest / 4);
+}
+
+function collapse(text) {
+  return String(text ?? "").replace(/[ \t\u00a0]+/g, " ").trim();
+}
+
+/**
+ * 把一段长文本按段落切，超长段落再按句末标点切。
+ * 不引入第三方分词库：中英混排下按「段落 → 句子 → 硬切」三级回退足够。
+ */
+function splitParagraphs(text) {
+  return String(text ?? "")
+    .split(/\n{2,}/)
+    .map((part) => part.trim())
+    .filter((part) => part !== "");
+}
+
+function splitSentences(text) {
+  // 句末标点后跟空白或直接跟非空白中文标点都算断句
+  const parts = String(text ?? "").split(/(?<=[。！？；.!?;])\s*/);
+  return parts.map((p) => p.trim()).filter((p) => p !== "");
+}
+
+function hardSplit(text, maxChars) {
+  const out = [];
+  let rest = text;
+  while (rest.length > maxChars) {
+    out.push(rest.slice(0, maxChars));
+    rest = rest.slice(maxChars);
+  }
+  if (rest !== "") out.push(rest);
+  return out;
+}
+
+/** 取一段文本的尾部作为下一块的重叠前缀（按句子对齐，不从字中间切）。 */
+function overlapTail(text, overlapChars) {
+  if (overlapChars <= 0) return "";
+  const s = String(text ?? "");
+  if (s.length <= overlapChars) return s;
+  const tail = s.slice(-overlapChars);
+  // 尽量从句子边界开始
+  const match = tail.match(/[。！？；.!?;]\s*/);
+  if (match && match.index !== undefined && match.index < overlapChars * 0.6) {
+    return tail.slice(match.index + match[0].length);
+  }
+  return tail;
+}
+
+/**
+ * 把一个段落序列打包成块。
+ * @param {{text: string, heading?: string, level?: number, page?: number|null, meta?: object}[]} blocks
+ */
+export function packBlocks(blocks, options = {}) {
+  const maxChars = Math.max(MIN_PART_CHARS, Math.round(Number(options.partChars) || DEFAULT_PART_CHARS));
+  const overlap = Math.max(0, Math.round(Number(options.overlapChars) ?? DEFAULT_OVERLAP_CHARS));
+  const minChars = Math.min(MIN_PART_CHARS, Math.floor(maxChars / 4));
+
+  const parts = [];
+  let buffer = "";
+  let bufferHeading = "";
+  let bufferLevel = 0;
+  let bufferPage = null;
+  let bufferMeta = null;
+
+  const flush = () => {
+    const text = collapse(buffer);
+    buffer = "";
+    if (text === "") return;
+    parts.push({
+      text,
+      heading: bufferHeading,
+      level: bufferLevel,
+      page: bufferPage,
+      meta: bufferMeta ?? {},
+      chars: text.length,
+      tokens: estimateTokens(text),
+    });
+    const tail = overlapTail(text, overlap);
+    buffer = tail;
+    bufferPage = bufferPage;
+  };
+
+  for (const block of blocks) {
+    const text = collapse(block?.text);
+    if (text === "") continue;
+    const heading = String(block?.heading ?? "");
+    const level = Number(block?.level) || 0;
+
+    // 标题变化：先把当前缓冲收口，避免「上一节的内容」混进下一节的块
+    const headingChanged = heading !== bufferHeading;
+    if (headingChanged && buffer.trim() !== "") flush();
+    if (headingChanged) {
+      bufferHeading = heading;
+      bufferLevel = level;
+      bufferPage = block?.page ?? null;
+      bufferMeta = block?.meta ?? null;
+    } else if (bufferPage === null && block?.page != null) {
+      bufferPage = block.page;
+    }
+
+    if (text.length > maxChars) {
+      if (buffer.trim() !== "") flush();
+      // 超长块：段落 → 句子 → 硬切
+      const paragraphs = splitParagraphs(text);
+      const units = paragraphs.flatMap((p) => (p.length > maxChars ? splitSentences(p) : [p]));
+      const finalUnits = units.flatMap((u) => (u.length > maxChars ? hardSplit(u, maxChars) : [u]));
+      let piece = "";
+      for (const unit of finalUnits) {
+        if (piece.length + unit.length + 2 > maxChars && piece.trim() !== "") {
+          buffer = piece;
+          bufferPage = block?.page ?? bufferPage;
+          flush();
+          piece = overlapTail(collapse(piece), overlap);
+        }
+        piece = piece === "" ? unit : `${piece}\n\n${unit}`;
+      }
+      if (piece.trim() !== "") {
+        buffer = buffer === "" ? piece : `${buffer}\n\n${piece}`;
+      }
+      continue;
+    }
+
+    if (buffer.trim() === "") {
+      buffer = text;
+    } else if (buffer.length + text.length + 2 > maxChars) {
+      flush();
+      buffer = buffer === "" ? text : `${buffer}\n\n${text}`;
+    } else {
+      buffer = `${buffer}\n\n${text}`;
+    }
+  }
+
+  // 收尾：最后一段不足 minChars 且已有前一块时，并进前一块
+  const tail = collapse(buffer);
+  if (tail !== "") {
+    if (parts.length > 0 && tail.length < minChars) {
+      parts[parts.length - 1].text = `${parts[parts.length - 1].text}\n\n${tail}`;
+      parts[parts.length - 1].chars = parts[parts.length - 1].text.length;
+      parts[parts.length - 1].tokens = estimateTokens(parts[parts.length - 1].text);
+    } else {
+      parts.push({
+        text: tail,
+        heading: bufferHeading,
+        level: bufferLevel,
+        page: bufferPage,
+        meta: bufferMeta ?? {},
+        chars: tail.length,
+        tokens: estimateTokens(tail),
+      });
+    }
+  }
+
+  // 再兜一层：单个碎片不足 minChars 的并入前一/后一块
+  const merged = [];
+  for (const part of parts) {
+    const prev = merged[merged.length - 1];
+    if (part.chars < minChars && prev && prev.chars + part.chars <= maxChars * 1.5) {
+      prev.text = `${prev.text}\n\n${part.text}`;
+      prev.chars = prev.text.length;
+      prev.tokens = estimateTokens(prev.text);
+      if (prev.page === null) prev.page = part.page;
+      continue;
+    }
+    merged.push(part);
+  }
+
+  return merged.map((part, index) => ({
+    ...part,
+    index: index + 1,
+  }));
+}
+
+/** 长文档的粗目录：取所有 level<=2 的标题作为大纲。 */
+export function outlineOf(parts) {
+  return parts
+    .filter((part) => part.heading && part.level > 0 && part.level <= 2)
+    .map((part) => ({ title: part.heading, level: part.level, page: part.page, part: part.index }))
+    .filter((item, index, list) => index === 0 || item.title !== list[index - 1].title);
+}