dsh-shared-kb 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -2
- package/cordis.patch.yml +11 -0
- package/lib/client.js +61 -0
- package/lib/core/atomic.js +87 -0
- package/lib/core/errors.js +39 -0
- package/lib/core/hubconfig.js +151 -0
- package/lib/core/paths.js +129 -0
- package/lib/icon.svg +7 -0
- package/lib/index.js +202 -0
- package/lib/indexer/embedder.js +325 -0
- package/lib/indexer/sqlite-index.js +431 -0
- package/lib/indexer/tokenize.js +151 -0
- package/lib/locale/zh-CN.json +4 -0
- package/lib/parse/chunk.js +202 -0
- package/lib/parse/parser.js +643 -0
- package/lib/search/search.js +194 -0
- package/lib/service/hub.js +791 -0
- package/lib/store/docs.js +927 -0
- package/lib/store/lease.js +251 -0
- package/lib/store/manifest.js +269 -0
- package/lib/tools/index.js +759 -0
- package/lib/ui/app.js +682 -0
- package/lib/ui/icon.svg +7 -0
- package/lib/ui/index.html +133 -0
- package/lib/ui/style.css +292 -0
- package/lib/web/api.js +225 -0
- package/lib/web/http.js +114 -0
- package/lib/web/server.js +149 -0
- package/package.json +83 -4
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 路径与命名规则。
|
|
3
|
+
*
|
|
4
|
+
* 铁律:库名只允许字母数字汉字与 - _ .,不许出现路径分隔符或 ..,
|
|
5
|
+
* 因为库名直接当目录名用。文档 id 由相对键的哈希派生,天然安全。
|
|
6
|
+
*/
|
|
7
|
+
import { createHash } from "node:crypto";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { basename, extname, isAbsolute, join, resolve, sep } from "node:path";
|
|
10
|
+
import { badRequest } from "./errors.js";
|
|
11
|
+
|
|
12
|
+
/** 展开 ~ 与 ~/xxx。 */
|
|
13
|
+
export function expandUser(input) {
|
|
14
|
+
const text = String(input ?? "");
|
|
15
|
+
if (text === "~") return homedir();
|
|
16
|
+
if (text.startsWith("~/") || text.startsWith("~\\")) return join(homedir(), text.slice(2));
|
|
17
|
+
return text;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** 把用户给的物理地址解析成绝对路径;相对路径按 cwd 解析。 */
|
|
21
|
+
export function resolveRoot(input, cwd = process.cwd()) {
|
|
22
|
+
const expanded = expandUser(String(input ?? "").trim());
|
|
23
|
+
if (expanded === "") throw badRequest("kbRoot 不能为空");
|
|
24
|
+
return isAbsolute(expanded) ? resolve(expanded) : resolve(cwd, expanded);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const KB_NAME_SAFE = /^[\p{L}\p{N}][\p{L}\p{N}._-]{0,63}$/u;
|
|
28
|
+
|
|
29
|
+
/** 库名规范化:去空白、禁止路径分隔符与保留名。 */
|
|
30
|
+
export function sanitizeKbName(input) {
|
|
31
|
+
const raw = String(input ?? "").trim();
|
|
32
|
+
if (raw === "") throw badRequest("库名不能为空");
|
|
33
|
+
const name = raw.normalize("NFC");
|
|
34
|
+
if (name.includes("/") || name.includes("\\") || name.includes("..")) {
|
|
35
|
+
throw badRequest(`库名不允许包含路径分隔符或 "..":${raw}`);
|
|
36
|
+
}
|
|
37
|
+
if (name.startsWith(".")) throw badRequest(`库名不允许以 . 开头(保留给 .hub/.index 等内部目录):${raw}`);
|
|
38
|
+
if (!KB_NAME_SAFE.test(name)) {
|
|
39
|
+
throw badRequest(`库名只能包含字母、数字、汉字、- _ .,且不超过 64 字符:${raw}`);
|
|
40
|
+
}
|
|
41
|
+
return name;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function hubDir(root) {
|
|
45
|
+
return join(root, ".hub");
|
|
46
|
+
}
|
|
47
|
+
export function kbRootDir(root, name) {
|
|
48
|
+
return join(root, name);
|
|
49
|
+
}
|
|
50
|
+
export function kbMetaPath(dir) {
|
|
51
|
+
return join(dir, "kb.json");
|
|
52
|
+
}
|
|
53
|
+
export function sourcesDir(dir) {
|
|
54
|
+
return join(dir, "sources");
|
|
55
|
+
}
|
|
56
|
+
export function markdownDir(dir) {
|
|
57
|
+
return join(dir, "markdown");
|
|
58
|
+
}
|
|
59
|
+
export function docDir(dir, docId) {
|
|
60
|
+
return join(markdownDir(dir), docId);
|
|
61
|
+
}
|
|
62
|
+
export function docMetaPath(dir, docId) {
|
|
63
|
+
return join(docDir(dir, docId), "meta.json");
|
|
64
|
+
}
|
|
65
|
+
export function docMergedPath(dir, docId) {
|
|
66
|
+
return join(docDir(dir, docId), `${docId}.md`);
|
|
67
|
+
}
|
|
68
|
+
export function notesDir(dir) {
|
|
69
|
+
return join(dir, "notes");
|
|
70
|
+
}
|
|
71
|
+
export function exportsDir(dir) {
|
|
72
|
+
return join(dir, "exports");
|
|
73
|
+
}
|
|
74
|
+
export function snapshotsDir(dir) {
|
|
75
|
+
return join(dir, "snapshots");
|
|
76
|
+
}
|
|
77
|
+
export function locksDir(dir) {
|
|
78
|
+
return join(dir, ".locks");
|
|
79
|
+
}
|
|
80
|
+
export function localIndexDir(dir) {
|
|
81
|
+
return join(dir, ".index");
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** 文档 id:相对键(导入时的路径或标题)的短哈希,同键必然同 id。 */
|
|
85
|
+
export function docIdFor(relKey) {
|
|
86
|
+
return createHash("sha256").update(String(relKey), "utf8").digest("hex").slice(0, 16);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** 笔记 id:标题派生,同标题必然同 id(重复写=更新而不是新建)。 */
|
|
90
|
+
export function noteIdFor(title) {
|
|
91
|
+
return "n" + createHash("sha256").update(String(title).trim(), "utf8").digest("hex").slice(0, 15);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** 把源文件名整理成可安全落盘的形式(保留可读性,去掉非法字符)。 */
|
|
95
|
+
export function safeFileName(input, fallback = "file") {
|
|
96
|
+
const raw = basename(String(input ?? "").trim()) || fallback;
|
|
97
|
+
const cleaned = raw
|
|
98
|
+
.replace(/[\u0000-\u001f<>:"/\\|?*]/g, "_")
|
|
99
|
+
.replace(/\s+/g, " ")
|
|
100
|
+
.trim();
|
|
101
|
+
const limited = cleaned.length > 120 ? cleaned.slice(0, 120) : cleaned;
|
|
102
|
+
return limited === "" || limited === "." || limited === ".." ? fallback : limited;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** 判定路径是否落在某个根之内(防目录穿越)。 */
|
|
106
|
+
export function isInside(root, target) {
|
|
107
|
+
const a = resolve(root);
|
|
108
|
+
const b = resolve(target);
|
|
109
|
+
if (a === b) return true;
|
|
110
|
+
return b.startsWith(a.endsWith(sep) ? a : a + sep);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const TEXTUAL_EXTS = new Set([".txt", ".md", ".markdown", ".json", ".csv", ".log", ".yml", ".yaml"]);
|
|
114
|
+
|
|
115
|
+
export function extOf(path) {
|
|
116
|
+
return extname(String(path ?? "")).toLowerCase();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function isTextualExt(ext) {
|
|
120
|
+
return TEXTUAL_EXTS.has(ext);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** 把相对键统一成 / 分隔,跨平台一致(用于 docId 与展示)。 */
|
|
124
|
+
export function normalizeRelKey(input) {
|
|
125
|
+
return String(input ?? "")
|
|
126
|
+
.split(/[\\/]+/)
|
|
127
|
+
.filter((part) => part !== "" && part !== ".")
|
|
128
|
+
.join("/");
|
|
129
|
+
}
|
package/lib/icon.svg
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round">
|
|
2
|
+
<path d="M4 5.5A2.5 2.5 0 0 1 6.5 3H19a1 1 0 0 1 1 1v13.5"/>
|
|
3
|
+
<path d="M4 5.5V19a2 2 0 0 0 2 2h12.5"/>
|
|
4
|
+
<path d="M6 17h13"/>
|
|
5
|
+
<circle cx="15.5" cy="9.5" r="2.5"/>
|
|
6
|
+
<path d="M15.5 12v4.5"/>
|
|
7
|
+
</svg>
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 插件入口(Host 半)。
|
|
3
|
+
*
|
|
4
|
+
* 生命周期:
|
|
5
|
+
* apply(ctx, config)
|
|
6
|
+
* → 解析物理根目录(kbRoot),确保 .hub 标记
|
|
7
|
+
* → 创建 hub 门面(懒按需,不在这里就连库连索引)
|
|
8
|
+
* → 注册 kb_* 工具
|
|
9
|
+
* → 注册 /shared-kb 工作台路由(宿主有 webServer 才做)
|
|
10
|
+
* → 注册一条 systemPrompt 说明(告诉模型这个库怎么用)
|
|
11
|
+
* → 返回 dispose:关闭所有 sqlite 句柄
|
|
12
|
+
*
|
|
13
|
+
* 硬约束:
|
|
14
|
+
* - 不在 apply 里做耗时工作(导入/建索引),否则拖慢宿主启动;
|
|
15
|
+
* - 任何失败都不能让宿主起不来:包一层 try/catch,失败降级为「不可用」
|
|
16
|
+
* 并在日志里写清楚,而不是 throw 出去。
|
|
17
|
+
*/
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
21
|
+
import z from "@deepseek-ai/schemastery";
|
|
22
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
23
|
+
import { createHub } from "./service/hub.js";
|
|
24
|
+
import { registerTools } from "./tools/index.js";
|
|
25
|
+
import { registerWeb } from "./web/server.js";
|
|
26
|
+
import { resolveRoot } from "./core/paths.js";
|
|
27
|
+
|
|
28
|
+
const name = "shared-kb";
|
|
29
|
+
const inject = ["tools", "systemPrompt"];
|
|
30
|
+
|
|
31
|
+
const pluginRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
|
32
|
+
|
|
33
|
+
/** 读 package.json 版本(发版核对时日志里要有版本号)。 */
|
|
34
|
+
function pluginVersion() {
|
|
35
|
+
try {
|
|
36
|
+
return JSON.parse(readFileSync(join(pluginRoot, "package.json"), "utf8")).version ?? "0.0.0";
|
|
37
|
+
} catch {
|
|
38
|
+
return "0.0.0";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export const Config = z.object({
|
|
43
|
+
kbRoot: z
|
|
44
|
+
.string()
|
|
45
|
+
.default("~/.dsh/shared-kb")
|
|
46
|
+
.description("知识库物理根目录。每个库在它下面建一个独立子文件夹;可以指向共享盘(UNC 路径或已挂载的盘符)。"),
|
|
47
|
+
defaultKb: z.string().default("default").description("未指定 kb 时使用的默认库名。"),
|
|
48
|
+
semanticProvider: z
|
|
49
|
+
.string()
|
|
50
|
+
.default("auto")
|
|
51
|
+
.description("语义检索 provider:auto(优先 onnx,缺失回退 builtin)/ onnx / remote / builtin。"),
|
|
52
|
+
semanticModelDir: z
|
|
53
|
+
.string()
|
|
54
|
+
.default("")
|
|
55
|
+
.description("ONNX 模型目录(需含 model.onnx / tokenizer.json / tokenizer_config.json)。留空则 onnx 不可用。"),
|
|
56
|
+
remoteBaseUrl: z.string().default("").description("remote provider:OpenAI 兼容 base URL(如 https://api.openai.com/v1)。"),
|
|
57
|
+
remoteApiKey: z.string().default("").description("remote provider:API Key。"),
|
|
58
|
+
remoteModel: z.string().default("").description("remote provider:embedding 模型名。"),
|
|
59
|
+
remoteDim: z.number().default(1024).description("remote provider 的向量维度(需与模型实际一致)。"),
|
|
60
|
+
builtinDim: z.number().default(512).description("builtin provider 的向量维度。"),
|
|
61
|
+
partChars: z.number().default(1200).description("每块字符数上限。"),
|
|
62
|
+
overlapChars: z.number().default(150).description("相邻块的重叠字符数。"),
|
|
63
|
+
indexCacheRoot: z
|
|
64
|
+
.string()
|
|
65
|
+
.default("")
|
|
66
|
+
.description("本机索引缓存根目录。留空用 ~/.dsh/shared-kb/indexes/<根目录哈希>。索引不放进共享盘(SQLite 在 SMB 上不可靠)。"),
|
|
67
|
+
historyKeep: z.number().default(10).description("每篇文档保留多少个历史正文版本。"),
|
|
68
|
+
maxImportBytes: z.number().default(536870912).description("单文件导入上限(字节,默认 512 MiB)。"),
|
|
69
|
+
autoIndex: z.boolean().default(true).description("导入后与检索前自动增量同步索引。"),
|
|
70
|
+
webEnabled: z.boolean().default(true).description("是否挂载 /shared-kb 工作台。"),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
export { name, inject };
|
|
74
|
+
|
|
75
|
+
export function apply(ctx, config = {}) {
|
|
76
|
+
const version = pluginVersion();
|
|
77
|
+
const log = (message) => {
|
|
78
|
+
try {
|
|
79
|
+
ctx.logger?.info?.(`[shared-kb] ${message}`);
|
|
80
|
+
} catch {
|
|
81
|
+
/* 日志不可用不应影响功能 */
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
const warn = (message) => {
|
|
85
|
+
try {
|
|
86
|
+
ctx.logger?.warn?.(`[shared-kb] ${message}`);
|
|
87
|
+
} catch {
|
|
88
|
+
/* ignore */
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
let hub = null;
|
|
93
|
+
let hubPromise = null;
|
|
94
|
+
const disposers = [];
|
|
95
|
+
|
|
96
|
+
/** 懒创建 hub:第一次真正用到时才连库。 */
|
|
97
|
+
const getHub = () => {
|
|
98
|
+
if (hub) return Promise.resolve(hub);
|
|
99
|
+
if (!hubPromise) {
|
|
100
|
+
const root = resolveRoot(config.kbRoot ?? "~/.dsh/shared-kb");
|
|
101
|
+
hubPromise = createHub({
|
|
102
|
+
root,
|
|
103
|
+
kbRoot: root,
|
|
104
|
+
defaultKb: config.defaultKb ?? "default",
|
|
105
|
+
semanticProvider: config.semanticProvider ?? "auto",
|
|
106
|
+
semanticModelDir: config.semanticModelDir ?? "",
|
|
107
|
+
remoteBaseUrl: config.remoteBaseUrl ?? "",
|
|
108
|
+
remoteApiKey: config.remoteApiKey ?? "",
|
|
109
|
+
remoteModel: config.remoteModel ?? "",
|
|
110
|
+
remoteDim: Number(config.remoteDim) || 1024,
|
|
111
|
+
builtinDim: Number(config.builtinDim) || 512,
|
|
112
|
+
partChars: Number(config.partChars) || 1200,
|
|
113
|
+
overlapChars: Number(config.overlapChars) ?? 150,
|
|
114
|
+
indexCacheRoot: config.indexCacheRoot ? resolveRoot(config.indexCacheRoot) : "",
|
|
115
|
+
historyKeep: Number(config.historyKeep) || 10,
|
|
116
|
+
maxImportBytes: Number(config.maxImportBytes) || 512 * 1024 * 1024,
|
|
117
|
+
autoIndex: config.autoIndex !== false,
|
|
118
|
+
logger: { warn },
|
|
119
|
+
})
|
|
120
|
+
.then((instance) => {
|
|
121
|
+
hub = instance;
|
|
122
|
+
log(`已就绪 v${version},根目录 ${instance.root},索引 ${instance.indexCacheRoot}`);
|
|
123
|
+
return instance;
|
|
124
|
+
})
|
|
125
|
+
.catch((error) => {
|
|
126
|
+
hubPromise = null;
|
|
127
|
+
warn(`初始化失败:${String(error?.message ?? error)}`);
|
|
128
|
+
throw error;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
return hubPromise;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// 工具注册需要一个同步的 hub 句柄:这里给一个代理对象,
|
|
135
|
+
// 每个方法调用时才 await getHub(),这样工具注册本身不阻塞启动。
|
|
136
|
+
const lazyHub = new Proxy(
|
|
137
|
+
{},
|
|
138
|
+
{
|
|
139
|
+
get(_target, prop) {
|
|
140
|
+
if (prop === "root") return resolveRoot(config.kbRoot ?? "~/.dsh/shared-kb");
|
|
141
|
+
if (prop === "then") return undefined;
|
|
142
|
+
return async (...args) => {
|
|
143
|
+
const instance = await getHub();
|
|
144
|
+
const fn = instance[prop];
|
|
145
|
+
if (typeof fn !== "function") throw new Error(`hub 没有方法 ${String(prop)}`);
|
|
146
|
+
return fn.apply(instance, args);
|
|
147
|
+
};
|
|
148
|
+
},
|
|
149
|
+
}
|
|
150
|
+
);
|
|
151
|
+
// 有几个同步属性(providerInfo / indexCacheRoot)在工具里当值用,单独兜住
|
|
152
|
+
lazyHub.providerInfo = () => (hub ? hub.providerInfo() : { name: "未初始化", dim: 0, kind: "unknown" });
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
const { count } = registerTools(ctx, { hub: lazyHub, defineTool, config });
|
|
156
|
+
log(`注册 ${count} 个工具`);
|
|
157
|
+
} catch (error) {
|
|
158
|
+
warn(`工具注册失败:${String(error?.message ?? error)}`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// 工作台:宿主提供 webServer 才挂,desktop/headless 不受影响
|
|
162
|
+
if (config.webEnabled !== false) {
|
|
163
|
+
try {
|
|
164
|
+
const dispose = registerWeb(ctx, { hub: lazyHub, getHub, config, version });
|
|
165
|
+
if (typeof dispose === "function") disposers.push(dispose);
|
|
166
|
+
} catch (error) {
|
|
167
|
+
warn(`工作台挂载失败:${String(error?.message ?? error)}`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// 系统提示:只描述能力边界与使用顺序,不塞长文
|
|
172
|
+
try {
|
|
173
|
+
const prompt = [
|
|
174
|
+
"知识库(dsh-shared-kb):",
|
|
175
|
+
`- 物理根目录 ${resolveRoot(config.kbRoot ?? "~/.dsh/shared-kb")},每个库一个独立子文件夹。`,
|
|
176
|
+
"- 需要库内资料时:先 skb_query 检索(中文短语直接写;标准号写 GB/T 或 GBT 都行),命中后用 skb_read 精读对应块。",
|
|
177
|
+
"- 导入资料用 skb_import(支持 pdf/docx/xlsx/pptx/epub/md/txt/csv/html;目录可批量)。内容相同会自动跳过。",
|
|
178
|
+
"- 沉淀结论用 skb_note(同一标题再次写入=更新,不要另起「续」)。",
|
|
179
|
+
"- 索引在本机、资料在共享盘:多人多机同时用同一库时,写入靠 rev 号做并发保护;遇到 REV_CONFLICT 表示别人先改了,重新读取后再提交,不要盲目重试。",
|
|
180
|
+
].join("\n");
|
|
181
|
+
ctx.systemPrompt?.section?.({ name: "shared-kb", text: prompt, order: 60 });
|
|
182
|
+
} catch (error) {
|
|
183
|
+
// 老版本宿主没有 systemPrompt.section 时静默跳过
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return () => {
|
|
187
|
+
for (const dispose of disposers) {
|
|
188
|
+
try {
|
|
189
|
+
dispose();
|
|
190
|
+
} catch {
|
|
191
|
+
/* ignore */
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
hub?.close?.();
|
|
196
|
+
} catch {
|
|
197
|
+
/* ignore */
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export default { name, inject, Config, apply };
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding provider 抽象。
|
|
3
|
+
*
|
|
4
|
+
* 三种实现,按「能自动降级」的顺序尝试:
|
|
5
|
+
*
|
|
6
|
+
* 1. builtin —— 零依赖的本地哈希嵌入(character n-gram + 线性投影)。
|
|
7
|
+
* 不是真正语义模型,但对「同一份资料里的相近表述」有稳定的
|
|
8
|
+
* 向量表示,可在无任何模型文件时让语义检索链路保持可用,
|
|
9
|
+
* 也让索引格式在切换模型后仍然兼容(维度写进 meta)。
|
|
10
|
+
* 2. onnx —— onnxruntime-node + @huggingface/tokenizers 加载
|
|
11
|
+
* BGE 类中文模型(可选 peer,未装则跳过)。
|
|
12
|
+
* 3. remote —— 走 OpenAI 兼容 /v1/embeddings 接口(用户自备 key)。
|
|
13
|
+
*
|
|
14
|
+
* 关键约定:
|
|
15
|
+
* - 每个向量都带 provider 名与维度;索引 meta 记录它们,切换 provider 时
|
|
16
|
+
* 维度或名字变了就整体重建,绝不混用不同模型的向量(混用后相似度没有意义)。
|
|
17
|
+
* - 所有向量 L2 归一化后再存,检索用点积即余弦,省一次除法。
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
|
|
22
|
+
export const BUILTIN_DIM = 512;
|
|
23
|
+
export const BUILTIN_PROVIDER = "builtin-hash-v1";
|
|
24
|
+
|
|
25
|
+
/** L2 归一化(原地不修改入参)。 */
|
|
26
|
+
export function normalizeVector(vector) {
|
|
27
|
+
const out = new Float32Array(vector.length);
|
|
28
|
+
let sum = 0;
|
|
29
|
+
for (let i = 0; i < vector.length; i += 1) sum += vector[i] * vector[i];
|
|
30
|
+
const norm = Math.sqrt(sum);
|
|
31
|
+
if (norm === 0) return out;
|
|
32
|
+
for (let i = 0; i < vector.length; i += 1) out[i] = vector[i] / norm;
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function dot(a, b) {
|
|
37
|
+
const n = Math.min(a.length, b.length);
|
|
38
|
+
let sum = 0;
|
|
39
|
+
for (let i = 0; i < n; i += 1) sum += a[i] * b[i];
|
|
40
|
+
return sum;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* builtin 嵌入:hash trick + 字符 n-gram。
|
|
45
|
+
*
|
|
46
|
+
* 做法:对文本的汉字二元组/英文词做 feature hashing 落到 DIM 维,
|
|
47
|
+
* 每个特征带符号(正负哈希,抵消碰撞偏置),再按 TF 加权。
|
|
48
|
+
* 这是经典的 hashing vectorizer,检索里常被用作「零依赖兜底」。
|
|
49
|
+
*/
|
|
50
|
+
export function builtinEmbed(text, { dim = BUILTIN_DIM } = {}) {
|
|
51
|
+
const vector = new Float32Array(dim);
|
|
52
|
+
const source = String(text ?? "").normalize("NFKC").toLowerCase();
|
|
53
|
+
if (source.trim() === "") return vector;
|
|
54
|
+
|
|
55
|
+
const bump = (feature, weight) => {
|
|
56
|
+
const hash = createHash("sha1").update(feature, "utf8").digest();
|
|
57
|
+
const idx = (hash[0] | (hash[1] << 8) | (hash[2] << 16)) % dim;
|
|
58
|
+
const sign = (hash[3] & 1) === 0 ? 1 : -1;
|
|
59
|
+
vector[idx] += sign * weight;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
// 汉字 2-gram / 3-gram
|
|
63
|
+
const cjkRuns = source.match(/[\u3400-\u4dbf\u4e00-\u9fff]+/g) ?? [];
|
|
64
|
+
for (const run of cjkRuns) {
|
|
65
|
+
for (let i = 0; i + 1 < run.length; i += 1) bump(run.slice(i, i + 2), 1);
|
|
66
|
+
for (let i = 0; i + 2 < run.length; i += 2) bump(run.slice(i, i + 3), 0.6);
|
|
67
|
+
if (run.length === 1) bump(run, 1.2);
|
|
68
|
+
}
|
|
69
|
+
// 英文/数字词
|
|
70
|
+
for (const word of source.match(/[a-z][a-z0-9_-]{1,30}/g) ?? []) bump(`w:${word}`, 1);
|
|
71
|
+
for (const num of source.match(/\d+(?:\.\d+)*/g) ?? []) bump(`n:${num}`, 1.2);
|
|
72
|
+
|
|
73
|
+
return normalizeVector(vector);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** provider 工厂:返回 { name, dim, embed(texts) } */
|
|
77
|
+
export function createBuiltinProvider({ dim = BUILTIN_DIM } = {}) {
|
|
78
|
+
return {
|
|
79
|
+
name: `${BUILTIN_PROVIDER}`,
|
|
80
|
+
dim,
|
|
81
|
+
kind: "builtin",
|
|
82
|
+
async embed(texts) {
|
|
83
|
+
return (Array.isArray(texts) ? texts : [texts]).map((text) => builtinEmbed(text, { dim }));
|
|
84
|
+
},
|
|
85
|
+
async probe() {
|
|
86
|
+
return { ok: true, provider: BUILTIN_PROVIDER, dim };
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export const DEFAULT_MODEL_DIR = "";
|
|
92
|
+
export const ONNX_MODEL_NAME = "BAAI/bge-small-zh-v1.5";
|
|
93
|
+
export const ONNX_DIM = 512;
|
|
94
|
+
export const MAX_TOKENS = 512;
|
|
95
|
+
/** 查询侧指令(BGE 系列要求;文档侧不加)。 */
|
|
96
|
+
export const QUERY_PREFIX = "为这个句子生成表示以用于检索相关文章:";
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* ONNX 运行时:懒加载,模型或依赖缺失就返回 { available:false, reason }。
|
|
100
|
+
* 与 dsh-knowledge-wiki 的做法一致(同源模型、同源池化),但独立实现,
|
|
101
|
+
* 因为它要能被单独测试(注入假 session),且不假设插件目录结构。
|
|
102
|
+
*/
|
|
103
|
+
export function createOnnxProvider({
|
|
104
|
+
modelDir = "",
|
|
105
|
+
loadModule = (spec) => import(spec),
|
|
106
|
+
readFileFn = null,
|
|
107
|
+
logger = null,
|
|
108
|
+
dim = ONNX_DIM,
|
|
109
|
+
} = {}) {
|
|
110
|
+
let sessionPromise = null;
|
|
111
|
+
let tokenizerPromise = null;
|
|
112
|
+
let warned = false;
|
|
113
|
+
|
|
114
|
+
const warnOnce = (message) => {
|
|
115
|
+
if (warned) return;
|
|
116
|
+
warned = true;
|
|
117
|
+
try {
|
|
118
|
+
logger?.warn?.(message);
|
|
119
|
+
} catch {
|
|
120
|
+
/* ignore */
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
const init = async () => {
|
|
125
|
+
if (!modelDir) throw new Error("未配置语义模型目录");
|
|
126
|
+
const fs = readFileFn ?? (await import("node:fs/promises")).readFile;
|
|
127
|
+
const { join } = await import("node:path");
|
|
128
|
+
const ort = await loadModule("onnxruntime-node");
|
|
129
|
+
const { Tokenizer } = await loadModule("@huggingface/tokenizers");
|
|
130
|
+
const modelPath = join(modelDir, "model.onnx");
|
|
131
|
+
const tokenizerPath = join(modelDir, "tokenizer.json");
|
|
132
|
+
const session = await ort.InferenceSession.create(modelPath, {
|
|
133
|
+
executionProviders: ["cpu"],
|
|
134
|
+
graphOptimizationLevel: "all",
|
|
135
|
+
});
|
|
136
|
+
const tokenizerJson = JSON.parse(await fs(tokenizerPath, "utf8"));
|
|
137
|
+
const tokenizer = new Tokenizer(tokenizerJson, JSON.parse(await fs(join(modelDir, "tokenizer_config.json"), "utf8").catch(() => "{}")));
|
|
138
|
+
return { ort, session, tokenizer };
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const get = () => {
|
|
142
|
+
if (!sessionPromise) {
|
|
143
|
+
sessionPromise = init().catch((error) => {
|
|
144
|
+
sessionPromise = null;
|
|
145
|
+
throw error;
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
return sessionPromise;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const embedBatch = async (texts, withPrefix) => {
|
|
152
|
+
const { ort, session, tokenizer } = await get();
|
|
153
|
+
const list = (Array.isArray(texts) ? texts : [texts]).map((t) =>
|
|
154
|
+
withPrefix ? QUERY_PREFIX + String(t ?? "") : String(t ?? "")
|
|
155
|
+
);
|
|
156
|
+
const out = [];
|
|
157
|
+
const BATCH = 8;
|
|
158
|
+
for (let i = 0; i < list.length; i += BATCH) {
|
|
159
|
+
const slice = list.slice(i, i + BATCH);
|
|
160
|
+
const encoded = slice.map((text) => tokenizer.encode(text).getIds().slice(0, MAX_TOKENS));
|
|
161
|
+
const width = Math.max(...encoded.map((ids) => ids.length));
|
|
162
|
+
const ids = new BigInt64Array(slice.length * width);
|
|
163
|
+
const mask = new BigInt64Array(slice.length * width);
|
|
164
|
+
const types = new BigInt64Array(slice.length * width);
|
|
165
|
+
for (let r = 0; r < encoded.length; r += 1) {
|
|
166
|
+
for (let c = 0; c < width; c += 1) {
|
|
167
|
+
const value = encoded[r][c] ?? 0;
|
|
168
|
+
ids[r * width + c] = BigInt(value);
|
|
169
|
+
mask[r * width + c] = c < encoded[r].length ? 1n : 0n;
|
|
170
|
+
types[r * width + c] = 0n;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const feeds = {
|
|
174
|
+
input_ids: new ort.Tensor("int64", ids, [slice.length, width]),
|
|
175
|
+
attention_mask: new ort.Tensor("int64", mask, [slice.length, width]),
|
|
176
|
+
token_type_ids: new ort.Tensor("int64", types, [slice.length, width]),
|
|
177
|
+
};
|
|
178
|
+
const result = await session.run(feeds);
|
|
179
|
+
const hidden = result.last_hidden_state ?? Object.values(result)[0];
|
|
180
|
+
const [, seq, hiddenDim] = hidden.dims;
|
|
181
|
+
for (let r = 0; r < slice.length; r += 1) {
|
|
182
|
+
const vector = new Float32Array(hiddenDim);
|
|
183
|
+
// CLS 池化:第 0 个 token
|
|
184
|
+
for (let h = 0; h < hiddenDim; h += 1) vector[h] = hidden.data[r * seq * hiddenDim + h];
|
|
185
|
+
out.push(normalizeVector(vector));
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
name: `onnx:${ONNX_MODEL_NAME}`,
|
|
193
|
+
kind: "onnx",
|
|
194
|
+
dim,
|
|
195
|
+
async embed(texts) {
|
|
196
|
+
return embedBatch(texts, false);
|
|
197
|
+
},
|
|
198
|
+
async embedQuery(text) {
|
|
199
|
+
return (await embedBatch([text], true))[0];
|
|
200
|
+
},
|
|
201
|
+
async probe() {
|
|
202
|
+
try {
|
|
203
|
+
const v = await embedBatch(["探针"], false);
|
|
204
|
+
return { ok: true, provider: `onnx:${ONNX_MODEL_NAME}`, dim: v[0]?.length ?? 0 };
|
|
205
|
+
} catch (error) {
|
|
206
|
+
warnOnce(`语义模型不可用,已降级:${String(error?.message ?? error)}`);
|
|
207
|
+
return { ok: false, reason: String(error?.message ?? error) };
|
|
208
|
+
}
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* 远端 provider:OpenAI 兼容 /v1/embeddings。
|
|
215
|
+
* 批量请求 + 失败不重试(重试交给上层,避免雪崩)。
|
|
216
|
+
*/
|
|
217
|
+
export function createRemoteProvider({
|
|
218
|
+
baseUrl,
|
|
219
|
+
apiKey,
|
|
220
|
+
model,
|
|
221
|
+
dim = 1024,
|
|
222
|
+
fetchImpl = globalThis.fetch,
|
|
223
|
+
batchSize = 32,
|
|
224
|
+
timeoutMs = 30_000,
|
|
225
|
+
} = {}) {
|
|
226
|
+
if (!baseUrl) throw new Error("远端 embedding 需要 baseUrl");
|
|
227
|
+
if (!model) throw new Error("远端 embedding 需要 model");
|
|
228
|
+
const endpoint = String(baseUrl).replace(/\/+$/, "") + "/embeddings";
|
|
229
|
+
|
|
230
|
+
const call = async (input) => {
|
|
231
|
+
const controller = new AbortController();
|
|
232
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
233
|
+
try {
|
|
234
|
+
const response = await fetchImpl(endpoint, {
|
|
235
|
+
method: "POST",
|
|
236
|
+
headers: {
|
|
237
|
+
"content-type": "application/json",
|
|
238
|
+
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
|
|
239
|
+
},
|
|
240
|
+
body: JSON.stringify({ model, input }),
|
|
241
|
+
signal: controller.signal,
|
|
242
|
+
});
|
|
243
|
+
if (!response.ok) {
|
|
244
|
+
const text = await response.text().catch(() => "");
|
|
245
|
+
throw new Error(`embedding 接口返回 ${response.status}:${text.slice(0, 300)}`);
|
|
246
|
+
}
|
|
247
|
+
const payload = await response.json();
|
|
248
|
+
const rows = Array.isArray(payload?.data) ? payload.data : [];
|
|
249
|
+
if (rows.length !== input.length) {
|
|
250
|
+
throw new Error(`embedding 返回条数不匹配:期望 ${input.length},实际 ${rows.length}`);
|
|
251
|
+
}
|
|
252
|
+
return rows
|
|
253
|
+
.sort((a, b) => Number(a.index) - Number(b.index))
|
|
254
|
+
.map((row) => normalizeVector(Float32Array.from(row.embedding ?? [])));
|
|
255
|
+
} finally {
|
|
256
|
+
clearTimeout(timer);
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
return {
|
|
261
|
+
name: `remote:${model}`,
|
|
262
|
+
kind: "remote",
|
|
263
|
+
dim,
|
|
264
|
+
async embed(texts) {
|
|
265
|
+
const list = (Array.isArray(texts) ? texts : [texts]).map((t) => String(t ?? ""));
|
|
266
|
+
const out = [];
|
|
267
|
+
for (let i = 0; i < list.length; i += batchSize) {
|
|
268
|
+
out.push(...(await call(list.slice(i, i + batchSize))));
|
|
269
|
+
}
|
|
270
|
+
return out;
|
|
271
|
+
},
|
|
272
|
+
async probe() {
|
|
273
|
+
try {
|
|
274
|
+
const v = await call(["探针"]);
|
|
275
|
+
return { ok: true, provider: `remote:${model}`, dim: v[0]?.length ?? 0 };
|
|
276
|
+
} catch (error) {
|
|
277
|
+
return { ok: false, reason: String(error?.message ?? error) };
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* 按配置选 provider,并在首选不可用时降级到 builtin。
|
|
285
|
+
* 降级必须写进返回值,让上层能如实报告「现在用的是兜底嵌入」。
|
|
286
|
+
*/
|
|
287
|
+
export async function resolveProvider(config = {}) {
|
|
288
|
+
const mode = String(config.semanticProvider ?? "auto").toLowerCase();
|
|
289
|
+
const notes = [];
|
|
290
|
+
|
|
291
|
+
const tryProvider = async (provider) => {
|
|
292
|
+
const probe = await provider.probe();
|
|
293
|
+
if (probe.ok) return { provider, probe, notes };
|
|
294
|
+
notes.push(`${provider.name}: ${probe.reason}`);
|
|
295
|
+
return null;
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
if (mode === "remote") {
|
|
299
|
+
const provider = createRemoteProvider({
|
|
300
|
+
baseUrl: config.remoteBaseUrl,
|
|
301
|
+
apiKey: config.remoteApiKey,
|
|
302
|
+
model: config.remoteModel,
|
|
303
|
+
dim: Number(config.remoteDim) || 1024,
|
|
304
|
+
});
|
|
305
|
+
const ok = await tryProvider(provider);
|
|
306
|
+
if (ok) return ok;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
if (mode === "onnx" || mode === "auto") {
|
|
310
|
+
const provider = createOnnxProvider({
|
|
311
|
+
modelDir: config.semanticModelDir,
|
|
312
|
+
logger: config.logger,
|
|
313
|
+
});
|
|
314
|
+
const ok = await tryProvider(provider);
|
|
315
|
+
if (ok) return ok;
|
|
316
|
+
if (mode === "onnx") {
|
|
317
|
+
// 显式要求 onnx 但不可用:仍然降级,但把原因记清楚
|
|
318
|
+
notes.push("显式要求 onnx 但不可用,已降级为 builtin");
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const builtin = createBuiltinProvider({ dim: Number(config.builtinDim) || BUILTIN_DIM });
|
|
323
|
+
const ok = await tryProvider(builtin);
|
|
324
|
+
return ok ?? { provider: builtin, probe: { ok: true, provider: builtin.name, dim: builtin.dim }, notes };
|
|
325
|
+
}
|