hob-bot 1.0.4 → 1.2.0

package/server/src/knowledge-index.ts

@@ -0,0 +1,123 @@
+import { readFile } from "node:fs/promises";
+import type { KnowledgeChunk, KnowledgeEntry } from "../../shared/knowledge.ts";
+
+export type IndexedChunk = KnowledgeChunk & { start: number; end: number };
+export class UnsupportedIndex extends Error {}
+const MAX_TEXT = 5 * 1024 * 1024;
+// 中文 PDF 的文字层常把「手」「见」存成康熙部首、部首补充（Chrome、Word 导出都这样），查不到；
+// NFKC 能还原康熙部首，部首补充和「⼾」（NFKC 给的是繁体「戶」）按简体补上。
+const RADICALS: Record<string, string> = Object.fromEntries([..."⼾户⻅见⻆角⻓长⻔门⻚页⻛风⻜飞⻢马⻥鱼⻦鸟⻬齐⻮齿⻰龙⻳龟⻉贝⻋车⻘青⻙韦⻝食⻨麦⻩黄⻤鬼"].flatMap((c, i, all) => i % 2 ? [] : [[c, all[i + 1]!]]));
+
+/** PDF 抽出的文字：部首还原成汉字，去掉汉字之间的折行（不然跨行的词查不到）。PDF 只按页定位，位置变了不影响。 */
+export function cleanPdfText(text: string): string {
+  return text.replace(/[\u2E80-\u2FDF\uF900-\uFAFF]/g, (c) => RADICALS[c] ?? c.normalize("NFKC")).replace(/([\u3400-\u9fff])\n(?=[\u3400-\u9fff])/g, "$1");
+}
+
+/** 位置针对抽出的文字；PDF 每页独立计算。 */
+export async function extractChunks(entry: KnowledgeEntry): Promise<IndexedChunk[]> {
+  if (!["markdown", "text", "code", "csv", "json", "html", "xml", "pdf"].includes(entry.kind)) throw new UnsupportedIndex("这类文件还不能建索引");
+  const data = await readFile(entry.path);
+  const chunks: IndexedChunk[] = [];
+  let bytes = 0;
+  const add = (text: string, heading: string, start: number, page?: number) => {
+    const clean = text.trim();
+    if (!clean) return;
+    const offset = text.indexOf(clean);
+    chunks.push({ index: chunks.length, heading, text: clean, start: start + offset, end: start + offset + clean.length, ...(page ? { page } : {}) });
+  };
+  const split = (text: string, heading: string, start = 0, page?: number, prefix = "") => {
+    let pos = 0;
+    while (pos < text.length) {
+      let end = Math.min(pos + Math.max(1, 800 - prefix.length), text.length);
+      if (end < text.length) {
+        const part = text.slice(pos, end);
+        const boundary = Math.max(part.lastIndexOf("\n\n"), part.lastIndexOf("\n"), part.lastIndexOf("。"));
+        if (boundary >= 300) end = pos + boundary + 1;
+      }
+      add(prefix + text.slice(pos, end), heading, start + pos, page);
+      pos = end;
+    }
+  };
+  const sections = (text: string, page?: number) => {
+    bytes += Buffer.byteLength(text);
+    if (bytes > MAX_TEXT) throw new Error("文件太大：抽出的文字超过 5 MB");
+    const headings: string[] = [];
+    const pattern = entry.kind === "html" ? /^[ \t]{0,3}(#{1,6})[ \t]*(.+?)[ \t]*#*[ \t]*$/gm : /^[ \t]{0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$/gm;
+    let sectionStart = 0;
+    const base = page ? [entry.name, `第 ${page} 页`] : [entry.name];
+    let heading = base.join(" › ");
+    for (const match of text.matchAll(pattern)) {
+      const at = match.index;
+      split(text.slice(sectionStart, at), heading, sectionStart, page);
+      const level = match[1]!.length;
+      headings.length = level - 1;
+      headings[level - 1] = match[2]!.trim();
+      heading = [...base, ...headings.filter(Boolean)].join(" › ");
+      sectionStart = at + match[0].length;
+    }
+    split(text.slice(sectionStart), heading, sectionStart, page);
+  };
+  if (entry.kind === "pdf") {
+    const { extractText } = await import("unpdf");
+    const result = await extractText(new Uint8Array(data), { mergePages: false });
+    const pages = Array.isArray(result.text) ? result.text : [result.text];
+    if (!pages.some((p) => p.trim())) throw new UnsupportedIndex("没有文字层，三期识字");
+    pages.forEach((page, i) => sections(cleanPdfText(page), i + 1));
+  } else {
+    let text = data.toString("utf8");
+    if (Buffer.byteLength(text) > MAX_TEXT) throw new Error("文件太大：抽出的文字超过 5 MB");
+    if (entry.kind === "html") {
+      const blank = (value: string) => value.replace(/[^\n]/g, " ");
+      text = text.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, blank)
+        .replace(/<h([1-3])\b[^>]*>/gi, (tag, level: string) => `\n${"#".repeat(Number(level))}${" ".repeat(Math.max(0, tag.length - Number(level) - 1))}`)
+        .replace(/<\/h[1-3]\s*>/gi, (tag) => `\n${" ".repeat(tag.length - 1)}`)
+        .replace(/<[^>]*>/g, blank);
+      sections(text);
+    } else if (entry.kind === "markdown") sections(text);
+    else if (entry.kind === "csv") {
+      const ends: number[] = [];
+      let quoted = false;
+      for (let i = 0; i < text.length; i++) {
+        if (text[i] === '"') {
+          if (quoted && text[i + 1] === '"') i++;
+          else quoted = !quoted;
+        } else if (text[i] === "\n" && !quoted) ends.push(i + 1);
+      }
+      if (ends.at(-1) !== text.length) ends.push(text.length);
+      const first = ends.shift() ?? text.length;
+      const header = text.slice(0, first).trimEnd();
+      let start = first;
+      let end = first;
+      for (const rowEnd of ends) {
+        if (end > start && rowEnd - start + header.length + 1 > 800) {
+          chunks.push({ index: chunks.length, heading: entry.name, text: `${header}\n${text.slice(start, end).trimEnd()}`, start, end });
+          start = end;
+        }
+        end = rowEnd;
+      }
+      if (end > start) chunks.push({ index: chunks.length, heading: entry.name, text: `${header}\n${text.slice(start, end).trimEnd()}`, start, end });
+      else if (!chunks.length && header) add(header, entry.name, 0);
+    } else if (entry.kind === "json") {
+      JSON.parse(text);
+      const open = text.search(/\S/);
+      if (text[open] !== "{" && text[open] !== "[") split(text, entry.name);
+      else {
+        let start = open + 1, depth = 0, quoted = false, escaped = false;
+        for (let i = start; i < text.length; i++) {
+          const char = text[i]!;
+          if (quoted) {
+            if (escaped) escaped = false;
+            else if (char === "\\") escaped = true;
+            else if (char === '"') quoted = false;
+          } else if (char === '"') quoted = true;
+          else if (char === "{" || char === "[") depth++;
+          else if (char === "}" || char === "]") {
+            if (depth) depth--;
+            else { split(text.slice(start, i), entry.name, start); break; }
+          } else if (char === "," && depth === 0) { split(text.slice(start, i), entry.name, start); start = i + 1; }
+        }
+      }
+    } else split(text, entry.name);
+  }
+  return chunks;
+}

package/server/src/knowledge-tools.ts

@@ -0,0 +1,94 @@
+import { Type } from "@earendil-works/pi-ai";
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { knowledgeGroup, knowledgeLink, type KnowledgeCollection, type KnowledgeChunk, type KnowledgeEntry } from "../../shared/knowledge.ts";
+import type { HobRuntime } from "./runtime.ts";
+import { monthDayTimeText } from "./time.ts";
+
+export const KNOWLEDGE_INTRO = "（知识库里可能相关的内容。用得上就引用，并照抄出处链接；用不上就忽略。）";
+
+/** 知识库读不出来时当作没绑定：对话和任务照常，不被知识库拖垮。 */
+export async function boundKnowledge(runtime: HobRuntime, assistantId: string): Promise<KnowledgeCollection[]> {
+  const ids = runtime.store.assistant(assistantId)?.knowledge ?? [];
+  if (!ids.length) return [];
+  try {
+    const { collections } = await runtime.knowledge.list({ limit: 1 });
+    return collections.filter((collection) => ids.includes(collection.id));
+  } catch (error) {
+    console.error("读取知识库失败：", error);
+    return [];
+  }
+}
+
+export function formatKnowledge(results: { entry: KnowledgeEntry; chunk: KnowledgeChunk }[], max: number): string {
+  let remaining = max;
+  const lines: string[] = [];
+  for (const { entry, chunk } of results) {
+    const header = `${lines.length + 1}. ${chunk.heading}（[${entry.name}](${knowledgeLink(entry.id, chunk.index)})）\n   `;
+    const overhead = [...header].length + (lines.length ? 1 : 0);
+    if (remaining <= overhead) break;
+    const text = [...chunk.text].slice(0, remaining - overhead).join("");
+    if (!text) continue;
+    lines.push(header + text);
+    remaining -= overhead + [...text].length;
+  }
+  return lines.join("\n");
+}
+
+export function searchKnowledgeTool(runtime: HobRuntime, assistantId: string): ToolDefinition<any, any> {
+  return {
+    name: "search_knowledge",
+    label: "查知识库",
+    description: "在绑定给你的知识库里查内容，找出和问题最相关的几段原文，每段带出处链接。回答用到时照抄出处链接标明来源；查不到就直说知识库里没有，不要编。",
+    parameters: Type.Object({
+      query: Type.String({ description: "要查的问题或关键词，写成完整的一句效果更好。" }),
+      collection: Type.Optional(Type.String({ description: "可选，只查这个知识库，填名称。" })),
+    }),
+    execute: async (_id, params: { query: string; collection?: string }) => {
+      try {
+        const bound = await boundKnowledge(runtime, assistantId);
+        const selected = bound.find((collection) => collection.name === params.collection);
+        const results = await runtime.knowledge.search(params.query, selected ? [selected.id] : bound.map((collection) => collection.id), { limit: 6 });
+        const prefix = runtime.settings.settings.knowledge?.embedding.source === "openai" ? "" : "（只按关键词查）\n";
+        const intro = `${KNOWLEDGE_INTRO}\n`;
+        const formatted = formatKnowledge(results, 6000 - [...prefix, ...intro].length);
+        return { content: [{ type: "text", text: prefix + (formatted ? intro + formatted : "知识库里没有找到相关内容。") }], details: undefined };
+      } catch (error) {
+        return { content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }], details: undefined, isError: true };
+      }
+    },
+  };
+}
+
+export function findKnowledgeTool(runtime: HobRuntime): ToolDefinition<any, any> {
+  return {
+    name: "find_knowledge",
+    label: "找资料",
+    description: "在知识库里找文件：助手产出、用户在对话里发的素材和用户建的知识库都在里面。按关键词、类型和时间找，找到后用 read 打开。",
+    parameters: Type.Object({
+      query: Type.Optional(Type.String({ description: "可选，关键词，匹配名称、说明和文字内容；不填就按时间列最近的。" })),
+      type: Type.Optional(Type.Union([Type.Literal("image"), Type.Literal("document"), Type.Literal("table"), Type.Literal("page")], { description: "可选：image 图片、document 文档、table 表格、page 网页。" })),
+      days: Type.Optional(Type.Number({ description: "可选，只找最近几天的。", exclusiveMinimum: 0 })),
+    }),
+    execute: async (_id, params: { query?: string; type?: string; days?: number }) => {
+      try {
+        if (params.days !== undefined && (!Number.isFinite(params.days) || params.days <= 0)) throw new Error("days 必须是大于 0 的数字。");
+        const page = await runtime.knowledge.list({ q: params.query?.trim(), kind: params.type, limit: 21 });
+        const since = params.days === undefined ? -Infinity : Date.now() - params.days * 86400_000;
+        const entries = page.entries.filter((entry) => Date.parse(entry.updatedAt) >= since);
+        const labels = { image: "图片", document: "文档", table: "表格", page: "网页", other: "其他" };
+        const timezone = runtime.settings.settings.general.timezone;
+        const lines = entries.slice(0, 20).map((entry) => {
+          const assistant = entry.source?.assistantId;
+          const name = assistant ? runtime.store.assistant(assistant)?.name ?? assistant : "用户";
+          const line = `${entry.name}（${labels[knowledgeGroup(entry.kind)]}，${name}，${monthDayTimeText(timezone, new Date(entry.updatedAt))}）：${entry.path}`;
+          return entry.snippet ? `${line}\n  ${entry.snippet.replace(/\s+/g, " ").trim()}` : line;
+        });
+        if (!lines.length) lines.push("没有找到。");
+        if (entries.length > 20) lines.push("还有更多，换个关键词或缩短时间。");
+        return { content: [{ type: "text", text: lines.join("\n") }], details: undefined };
+      } catch (error) {
+        return { content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }], details: undefined, isError: true };
+      }
+    },
+  };
+}