@mushinjs/plugin-knowledge 0.0.0-stage → 0.2.0-next.20261113103537
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/index.d.ts +381 -0
- package/dist/index.js +1507 -0
- package/dist/index.js.map +1 -0
- package/dist/learn.d.ts +20 -0
- package/dist/learn.js +43 -0
- package/dist/learn.js.map +1 -0
- package/dist/node.js +129 -0
- package/dist/node.js.map +1 -0
- package/dist/search.d.ts +85 -0
- package/dist/search.js +218 -0
- package/dist/search.js.map +1 -0
- package/dist/text.d.ts +32 -0
- package/dist/text.js +183 -0
- package/dist/text.js.map +1 -0
- package/package.json +70 -3
- package/README.md +0 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,1507 @@
|
|
|
1
|
+
import { LEARN_PROMPT, candidateText, parseCandidates, transcript } from "./learn.js";
|
|
2
|
+
import { searchNode } from "./node.js";
|
|
3
|
+
import { SearchIndex, dot, normalize, passages, tokenize } from "./search.js";
|
|
4
|
+
import { FORMATS, chunkText, extractText, formatOf, htmlToText } from "./text.js";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { MushinError, Service, randomId, z } from "@mushinjs/core";
|
|
9
|
+
import { roleAtLeast } from "@mushinjs/plugin-bots";
|
|
10
|
+
// src/index.ts
|
|
11
|
+
const MAX_FILE_BYTES = 20971520;
|
|
12
|
+
const MAX_TEXT_CHARS = 2e6;
|
|
13
|
+
const NOTES_TITLE = "随手记";
|
|
14
|
+
const LEARNED_TITLE = "从对话学习";
|
|
15
|
+
const USAGE_KEY = "knowledge/learn-usage";
|
|
16
|
+
const LEARN_STATE_KEY = "knowledge/learn-state";
|
|
17
|
+
/** Fewest new messages worth a learning request. */
|
|
18
|
+
const LEARN_MIN_LINES = 5;
|
|
19
|
+
const DEFAULT_LEARN = {
|
|
20
|
+
enabled: false,
|
|
21
|
+
channels: [],
|
|
22
|
+
every: 50,
|
|
23
|
+
autoAccept: false,
|
|
24
|
+
dailyLimit: 20,
|
|
25
|
+
provider: "",
|
|
26
|
+
model: ""
|
|
27
|
+
};
|
|
28
|
+
const learnInput = z.object({
|
|
29
|
+
enabled: z.boolean().optional(),
|
|
30
|
+
channels: z.array(z.string().regex(/^(group|private):\S+$/)).optional().describe("\"group:<id>\" or \"private:<user id>\"; empty learns from every chat of the base's bots"),
|
|
31
|
+
every: z.number().int().min(10).max(500).optional().describe("Learn after this many new messages in a chat"),
|
|
32
|
+
autoAccept: z.boolean().optional().describe("Add what is learned straight away instead of waiting for review"),
|
|
33
|
+
dailyLimit: z.number().int().min(1).max(500).optional(),
|
|
34
|
+
provider: z.string().optional(),
|
|
35
|
+
model: z.string().optional()
|
|
36
|
+
});
|
|
37
|
+
const baseFields = {
|
|
38
|
+
name: z.string().trim().min(1).max(60),
|
|
39
|
+
description: z.string().max(500).optional(),
|
|
40
|
+
provider: z.string().optional().describe("Embedding provider id (providers.embedChoices); empty for keyword search only"),
|
|
41
|
+
model: z.string().optional().describe("Embedding model, e.g. text-embedding-3-small or bge-m3"),
|
|
42
|
+
chunkSize: z.number().int().min(100).max(4e3).optional().describe("Most characters per chunk (default 500)"),
|
|
43
|
+
chunkOverlap: z.number().int().min(0).max(1e3).optional().describe("Characters repeated from the previous chunk (default 50)"),
|
|
44
|
+
topK: z.number().int().min(1).max(20).optional(),
|
|
45
|
+
minScore: z.number().min(0).max(1).optional(),
|
|
46
|
+
bots: z.array(z.string()).optional().describe("Bot ids whose admins may add knowledge from chat, and whose chats it learns from"),
|
|
47
|
+
commands: z.object({
|
|
48
|
+
enabled: z.boolean().optional(),
|
|
49
|
+
role: z.enum([
|
|
50
|
+
"user",
|
|
51
|
+
"admin",
|
|
52
|
+
"master"
|
|
53
|
+
]).optional()
|
|
54
|
+
}).optional(),
|
|
55
|
+
learn: learnInput.optional()
|
|
56
|
+
};
|
|
57
|
+
const localDay = (date = /* @__PURE__ */ new Date()) => `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`;
|
|
58
|
+
const vectorBytes = (vector) => vector ? vector.buffer.slice(vector.byteOffset, vector.byteOffset + vector.byteLength) : /* @__PURE__ */ new ArrayBuffer(0);
|
|
59
|
+
const vectorOf = (bytes) => {
|
|
60
|
+
if (!bytes || !bytes.byteLength) return void 0;
|
|
61
|
+
const buffer = bytes instanceof ArrayBuffer ? bytes : bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
62
|
+
return new Float32Array(buffer);
|
|
63
|
+
};
|
|
64
|
+
const hashOf = (data) => createHash("sha256").update(data).digest("hex");
|
|
65
|
+
const errorText = (error) => error instanceof Error ? error.message : String(error);
|
|
66
|
+
/** The rows of a chunk as actions return them: no vector bytes. */
|
|
67
|
+
function publicChunk({ vector, ...row }) {
|
|
68
|
+
return {
|
|
69
|
+
...row,
|
|
70
|
+
embedded: !!vector?.byteLength
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Knowledge bases: documents split into chunks, searched by keywords and (with an embedding
|
|
75
|
+
* model) by meaning. Flows search them with a node, the Agent with `kb.search`, admins add to
|
|
76
|
+
* them from chat, and they can learn candidate knowledge from chats for review.
|
|
77
|
+
*/
|
|
78
|
+
var Knowledge = class extends Service {
|
|
79
|
+
static inject = [
|
|
80
|
+
"storage",
|
|
81
|
+
"actions",
|
|
82
|
+
"llm",
|
|
83
|
+
"bots",
|
|
84
|
+
"model",
|
|
85
|
+
"database"
|
|
86
|
+
];
|
|
87
|
+
home;
|
|
88
|
+
indexes = /* @__PURE__ */ new Map();
|
|
89
|
+
queue = Promise.resolve();
|
|
90
|
+
abort = new AbortController();
|
|
91
|
+
/** Base id → `chat key` → messages since the last learning. */
|
|
92
|
+
counters = /* @__PURE__ */ new Map();
|
|
93
|
+
learning = /* @__PURE__ */ new Set();
|
|
94
|
+
/** Chat key → when it was last learned from, so the same messages are not read twice. */
|
|
95
|
+
learnState = {};
|
|
96
|
+
usage = {
|
|
97
|
+
day: "",
|
|
98
|
+
counts: {}
|
|
99
|
+
};
|
|
100
|
+
constructor(ctx) {
|
|
101
|
+
super(ctx, "knowledge");
|
|
102
|
+
this.home = ctx;
|
|
103
|
+
}
|
|
104
|
+
get logger() {
|
|
105
|
+
return this.home.logger("knowledge");
|
|
106
|
+
}
|
|
107
|
+
get db() {
|
|
108
|
+
return this.home.database;
|
|
109
|
+
}
|
|
110
|
+
async *[Service.init]() {
|
|
111
|
+
const model = this.home.model;
|
|
112
|
+
model.extend("knowledge_base", {
|
|
113
|
+
id: "string",
|
|
114
|
+
name: "string",
|
|
115
|
+
description: "string",
|
|
116
|
+
provider: "string",
|
|
117
|
+
model: "string",
|
|
118
|
+
dimension: "unsigned",
|
|
119
|
+
chunkSize: "unsigned",
|
|
120
|
+
chunkOverlap: "unsigned",
|
|
121
|
+
topK: "unsigned",
|
|
122
|
+
minScore: "double",
|
|
123
|
+
bots: "json",
|
|
124
|
+
commands: "json",
|
|
125
|
+
learn: "json",
|
|
126
|
+
createdAt: "double",
|
|
127
|
+
updatedAt: "double"
|
|
128
|
+
}, { primary: "id" });
|
|
129
|
+
model.extend("knowledge_doc", {
|
|
130
|
+
id: "string",
|
|
131
|
+
base: "string",
|
|
132
|
+
title: "string",
|
|
133
|
+
kind: "string",
|
|
134
|
+
source: "string",
|
|
135
|
+
format: "string",
|
|
136
|
+
status: "string",
|
|
137
|
+
error: "text",
|
|
138
|
+
hash: "string",
|
|
139
|
+
chars: "unsigned",
|
|
140
|
+
chunks: "unsigned",
|
|
141
|
+
createdAt: "double",
|
|
142
|
+
updatedAt: "double"
|
|
143
|
+
}, {
|
|
144
|
+
primary: "id",
|
|
145
|
+
indexes: [["base"]]
|
|
146
|
+
});
|
|
147
|
+
model.extend("knowledge_chunk", {
|
|
148
|
+
id: "string",
|
|
149
|
+
base: "string",
|
|
150
|
+
doc: "string",
|
|
151
|
+
seq: "unsigned",
|
|
152
|
+
title: "string",
|
|
153
|
+
text: "text",
|
|
154
|
+
vector: "binary",
|
|
155
|
+
edited: "boolean",
|
|
156
|
+
meta: "json",
|
|
157
|
+
createdAt: "double",
|
|
158
|
+
updatedAt: "double"
|
|
159
|
+
}, {
|
|
160
|
+
primary: "id",
|
|
161
|
+
indexes: [["base"], ["doc", "seq"]]
|
|
162
|
+
});
|
|
163
|
+
model.extend("knowledge_candidate", {
|
|
164
|
+
id: "unsigned",
|
|
165
|
+
base: "string",
|
|
166
|
+
bot: "string",
|
|
167
|
+
channel: "string",
|
|
168
|
+
question: "text",
|
|
169
|
+
answer: "text",
|
|
170
|
+
status: "string",
|
|
171
|
+
chunk: "string",
|
|
172
|
+
createdAt: "double",
|
|
173
|
+
updatedAt: "double"
|
|
174
|
+
}, {
|
|
175
|
+
autoInc: true,
|
|
176
|
+
indexes: [["base", "status"]]
|
|
177
|
+
});
|
|
178
|
+
this.learnState = await this.home.storage.get(LEARN_STATE_KEY) ?? {};
|
|
179
|
+
this.usage = await this.home.storage.get(USAGE_KEY) ?? {
|
|
180
|
+
day: "",
|
|
181
|
+
counts: {}
|
|
182
|
+
};
|
|
183
|
+
yield this.home.effect(() => () => this.abort.abort(), "knowledge jobs");
|
|
184
|
+
const unfinished = await this.db.get("knowledge_doc", { status: [
|
|
185
|
+
"pending",
|
|
186
|
+
"parsing",
|
|
187
|
+
"embedding"
|
|
188
|
+
] });
|
|
189
|
+
for (const doc of unfinished) this.enqueue(() => this.process(doc.id));
|
|
190
|
+
yield* this.defineActions();
|
|
191
|
+
yield* this.defineCommands();
|
|
192
|
+
yield this.home.on("bots/message", (record) => this.onMessage(record));
|
|
193
|
+
this.home.inject(["flows"], (ctx) => {
|
|
194
|
+
ctx.flows.defineNode(searchNode(this));
|
|
195
|
+
});
|
|
196
|
+
this.home.inject(["agent"], (ctx) => {
|
|
197
|
+
ctx.agent.defineSkill({
|
|
198
|
+
id: "answer-from-knowledge",
|
|
199
|
+
name: "根据知识库回答",
|
|
200
|
+
description: "先在知识库里查资料,再根据查到的内容回答,并说明出处",
|
|
201
|
+
whenToUse: "问题和已经整理进知识库的资料有关(群规、产品说明、常见问题等)",
|
|
202
|
+
instructions: [
|
|
203
|
+
"1. 用 kb.list 看有哪些知识库;用户指定了就只查那个。",
|
|
204
|
+
"2. 用 kb.search 搜索问题,必要时换几种说法再搜。",
|
|
205
|
+
"3. 只根据命中的片段回答,在句末标出片段编号和标题;没查到就直接说知识库里没有,不要编。",
|
|
206
|
+
"4. 发现资料过时或互相矛盾时,指出来,并建议用 kb.chunks.update 修改。"
|
|
207
|
+
].join("\n"),
|
|
208
|
+
tools: [
|
|
209
|
+
"kb.list",
|
|
210
|
+
"kb.search",
|
|
211
|
+
"kb.chunks.list"
|
|
212
|
+
],
|
|
213
|
+
scenes: ["knowledge"]
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
this.logger.info("knowledge bases are ready");
|
|
217
|
+
}
|
|
218
|
+
async list() {
|
|
219
|
+
return this.db.get("knowledge_base", {}, { sort: { createdAt: "asc" } });
|
|
220
|
+
}
|
|
221
|
+
async get(id) {
|
|
222
|
+
const [base] = await this.db.get("knowledge_base", { id });
|
|
223
|
+
if (!base) throw new MushinError("not_found", `Knowledge base "${id}" not found`);
|
|
224
|
+
return base;
|
|
225
|
+
}
|
|
226
|
+
/** Bases with their document, chunk and candidate counts, for the list page. */
|
|
227
|
+
async summaries() {
|
|
228
|
+
const bases = await this.list();
|
|
229
|
+
const docs = await this.db.get("knowledge_doc", {});
|
|
230
|
+
const pending = await this.db.get("knowledge_candidate", { status: "pending" }, ["base"]);
|
|
231
|
+
return Promise.all(bases.map(async (base) => {
|
|
232
|
+
const own = docs.filter((doc) => doc.base === base.id);
|
|
233
|
+
const index = await this.index(base.id);
|
|
234
|
+
return {
|
|
235
|
+
...base,
|
|
236
|
+
docs: own.length,
|
|
237
|
+
chunks: index.size,
|
|
238
|
+
embedded: base.dimension ? index.embedded : 0,
|
|
239
|
+
busy: own.some((doc) => doc.status !== "ready" && doc.status !== "error"),
|
|
240
|
+
candidates: pending.filter((item) => item.base === base.id).length
|
|
241
|
+
};
|
|
242
|
+
}));
|
|
243
|
+
}
|
|
244
|
+
/** Embeds a probe text to learn the model's vector length; fails with a readable error. */
|
|
245
|
+
async probe(provider, model) {
|
|
246
|
+
try {
|
|
247
|
+
const result = await this.home.llm.embed({
|
|
248
|
+
provider,
|
|
249
|
+
model,
|
|
250
|
+
texts: ["test"],
|
|
251
|
+
signal: AbortSignal.timeout(3e4)
|
|
252
|
+
});
|
|
253
|
+
if (!result.dimension) throw new Error("the model returned an empty vector");
|
|
254
|
+
return result.dimension;
|
|
255
|
+
} catch (error) {
|
|
256
|
+
throw new MushinError("invalid_input", `The embedding model "${model}" did not work: ${errorText(error)}`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
async create(input) {
|
|
260
|
+
const provider = input.provider ?? "";
|
|
261
|
+
const model = provider ? input.model?.trim() ?? "" : "";
|
|
262
|
+
if (provider && !model) throw new MushinError("invalid_input", "Choose an embedding model, or no provider for keyword search only");
|
|
263
|
+
const dimension = provider ? await this.probe(provider, model) : 0;
|
|
264
|
+
const now = Date.now();
|
|
265
|
+
const base = {
|
|
266
|
+
id: randomId(6),
|
|
267
|
+
name: input.name,
|
|
268
|
+
description: input.description ?? "",
|
|
269
|
+
provider,
|
|
270
|
+
model,
|
|
271
|
+
dimension,
|
|
272
|
+
chunkSize: input.chunkSize ?? 500,
|
|
273
|
+
chunkOverlap: input.chunkOverlap ?? 50,
|
|
274
|
+
topK: input.topK ?? 5,
|
|
275
|
+
minScore: input.minScore ?? .3,
|
|
276
|
+
bots: input.bots ?? [],
|
|
277
|
+
commands: {
|
|
278
|
+
enabled: true,
|
|
279
|
+
role: "admin",
|
|
280
|
+
...input.commands
|
|
281
|
+
},
|
|
282
|
+
learn: {
|
|
283
|
+
...DEFAULT_LEARN,
|
|
284
|
+
...input.learn
|
|
285
|
+
},
|
|
286
|
+
createdAt: now,
|
|
287
|
+
updatedAt: now
|
|
288
|
+
};
|
|
289
|
+
await this.db.upsert("knowledge_base", [base]);
|
|
290
|
+
this.logger.info("created knowledge base %s%s", base.id, model ? ` with ${model}` : "");
|
|
291
|
+
this.changed(base.id);
|
|
292
|
+
return base;
|
|
293
|
+
}
|
|
294
|
+
async update(id, patch) {
|
|
295
|
+
const current = await this.get(id);
|
|
296
|
+
const next = {
|
|
297
|
+
...current,
|
|
298
|
+
...patch.name !== void 0 ? { name: patch.name } : {},
|
|
299
|
+
...patch.description !== void 0 ? { description: patch.description } : {},
|
|
300
|
+
...patch.chunkSize !== void 0 ? { chunkSize: patch.chunkSize } : {},
|
|
301
|
+
...patch.chunkOverlap !== void 0 ? { chunkOverlap: patch.chunkOverlap } : {},
|
|
302
|
+
...patch.topK !== void 0 ? { topK: patch.topK } : {},
|
|
303
|
+
...patch.minScore !== void 0 ? { minScore: patch.minScore } : {},
|
|
304
|
+
...patch.bots !== void 0 ? { bots: patch.bots } : {},
|
|
305
|
+
commands: {
|
|
306
|
+
...current.commands,
|
|
307
|
+
...patch.commands
|
|
308
|
+
},
|
|
309
|
+
learn: {
|
|
310
|
+
...DEFAULT_LEARN,
|
|
311
|
+
...current.learn,
|
|
312
|
+
...patch.learn
|
|
313
|
+
},
|
|
314
|
+
updatedAt: Date.now()
|
|
315
|
+
};
|
|
316
|
+
const provider = patch.provider ?? current.provider;
|
|
317
|
+
const model = provider ? (patch.model ?? current.model).trim() : "";
|
|
318
|
+
const modelChanged = provider !== current.provider || model !== current.model;
|
|
319
|
+
if (modelChanged) {
|
|
320
|
+
if (provider && !model) throw new MushinError("invalid_input", "Choose an embedding model, or no provider for keyword search only");
|
|
321
|
+
next.provider = provider;
|
|
322
|
+
next.model = model;
|
|
323
|
+
next.dimension = provider ? await this.probe(provider, model) : 0;
|
|
324
|
+
}
|
|
325
|
+
await this.db.upsert("knowledge_base", [next]);
|
|
326
|
+
this.indexes.delete(id);
|
|
327
|
+
this.changed(id);
|
|
328
|
+
if (modelChanged) {
|
|
329
|
+
this.logger.info("knowledge base %s now uses %s", next.id, model || "keywords only");
|
|
330
|
+
if (provider) this.reindex(id);
|
|
331
|
+
else await this.db.set("knowledge_chunk", { base: id }, { vector: /* @__PURE__ */ new ArrayBuffer(0) });
|
|
332
|
+
}
|
|
333
|
+
return next;
|
|
334
|
+
}
|
|
335
|
+
async remove(id) {
|
|
336
|
+
const base = await this.get(id);
|
|
337
|
+
await this.db.remove("knowledge_chunk", { base: id });
|
|
338
|
+
await this.db.remove("knowledge_doc", { base: id });
|
|
339
|
+
await this.db.remove("knowledge_candidate", { base: id });
|
|
340
|
+
await this.db.remove("knowledge_base", { id });
|
|
341
|
+
await rm(this.fileDir(id), {
|
|
342
|
+
recursive: true,
|
|
343
|
+
force: true
|
|
344
|
+
});
|
|
345
|
+
this.indexes.delete(id);
|
|
346
|
+
this.logger.info("deleted knowledge base %s", base.id);
|
|
347
|
+
this.changed(id);
|
|
348
|
+
return { ok: true };
|
|
349
|
+
}
|
|
350
|
+
changed(base) {
|
|
351
|
+
this.home.emit("knowledge/changed", { base });
|
|
352
|
+
}
|
|
353
|
+
progress(progress) {
|
|
354
|
+
this.home.emit("knowledge/progress", progress);
|
|
355
|
+
}
|
|
356
|
+
fileDir(base) {
|
|
357
|
+
return join(this.home.storage.dir, "knowledge", base);
|
|
358
|
+
}
|
|
359
|
+
filePath(doc) {
|
|
360
|
+
return join(this.fileDir(doc.base), `${doc.id}.${doc.format}`);
|
|
361
|
+
}
|
|
362
|
+
async docs(base) {
|
|
363
|
+
await this.get(base);
|
|
364
|
+
return this.db.get("knowledge_doc", { base }, { sort: { createdAt: "asc" } });
|
|
365
|
+
}
|
|
366
|
+
async doc(id) {
|
|
367
|
+
const [doc] = await this.db.get("knowledge_doc", { id });
|
|
368
|
+
if (!doc) throw new MushinError("not_found", `Document "${id}" not found`);
|
|
369
|
+
return doc;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Adds a document: a file (`data` in base64 with its `name`), pasted `text`, or a `url` to fetch.
|
|
373
|
+
* It is parsed, split and embedded in the background; `knowledge/progress` reports how it goes.
|
|
374
|
+
*/
|
|
375
|
+
async addDoc(input) {
|
|
376
|
+
const base = await this.get(input.base);
|
|
377
|
+
let kind;
|
|
378
|
+
let format;
|
|
379
|
+
let bytes;
|
|
380
|
+
let title = input.name?.trim() ?? "";
|
|
381
|
+
let source = "";
|
|
382
|
+
if (input.url) {
|
|
383
|
+
const url = new URL(input.url);
|
|
384
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new MushinError("invalid_input", "Only http and https links can be added");
|
|
385
|
+
kind = "url";
|
|
386
|
+
source = url.href;
|
|
387
|
+
const fetched = await this.fetchPage(url.href);
|
|
388
|
+
format = fetched.format;
|
|
389
|
+
bytes = fetched.bytes;
|
|
390
|
+
title ||= fetched.title || url.hostname + url.pathname;
|
|
391
|
+
} else if (input.data !== void 0) {
|
|
392
|
+
if (!title) throw new MushinError("invalid_input", "Give the file's name");
|
|
393
|
+
const detected = formatOf(title);
|
|
394
|
+
if (!detected) throw new MushinError("invalid_input", `Cannot read "${title}"; supported: ${FORMATS.join(", ")}`);
|
|
395
|
+
kind = "file";
|
|
396
|
+
format = detected;
|
|
397
|
+
source = title;
|
|
398
|
+
bytes = Buffer.from(input.data, "base64");
|
|
399
|
+
title = title.replace(/\.[^.]+$/, "");
|
|
400
|
+
} else if (input.text !== void 0) {
|
|
401
|
+
if (!input.text.trim()) throw new MushinError("invalid_input", "The text is empty");
|
|
402
|
+
kind = "text";
|
|
403
|
+
format = "md";
|
|
404
|
+
bytes = new TextEncoder().encode(input.text);
|
|
405
|
+
title ||= input.text.trim().split("\n")[0].replace(/^#+\s*/, "").slice(0, 40);
|
|
406
|
+
} else throw new MushinError("invalid_input", "Give a file (name and data), text, or a url");
|
|
407
|
+
if (bytes.byteLength > MAX_FILE_BYTES) throw new MushinError("invalid_input", `Files can be at most ${MAX_FILE_BYTES / 1024 / 1024} MB`);
|
|
408
|
+
const hash = hashOf(bytes);
|
|
409
|
+
const [same] = await this.db.get("knowledge_doc", {
|
|
410
|
+
base: base.id,
|
|
411
|
+
hash
|
|
412
|
+
});
|
|
413
|
+
if (same) throw new MushinError("conflict", `"${same.title}" in this base has the same content`);
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
const doc = {
|
|
416
|
+
id: randomId(6),
|
|
417
|
+
base: base.id,
|
|
418
|
+
title,
|
|
419
|
+
kind,
|
|
420
|
+
source,
|
|
421
|
+
format,
|
|
422
|
+
status: "pending",
|
|
423
|
+
error: "",
|
|
424
|
+
hash,
|
|
425
|
+
chars: 0,
|
|
426
|
+
chunks: 0,
|
|
427
|
+
createdAt: now,
|
|
428
|
+
updatedAt: now
|
|
429
|
+
};
|
|
430
|
+
await mkdir(this.fileDir(base.id), { recursive: true });
|
|
431
|
+
await writeFile(this.filePath(doc), bytes);
|
|
432
|
+
await this.db.upsert("knowledge_doc", [doc]);
|
|
433
|
+
this.logger.info("added \"%s\" to knowledge base %s", doc.title, base.id);
|
|
434
|
+
this.changed(base.id);
|
|
435
|
+
this.enqueue(() => this.process(doc.id));
|
|
436
|
+
return doc;
|
|
437
|
+
}
|
|
438
|
+
async fetchPage(url) {
|
|
439
|
+
let response;
|
|
440
|
+
try {
|
|
441
|
+
response = await fetch(url, {
|
|
442
|
+
signal: AbortSignal.timeout(2e4),
|
|
443
|
+
headers: { "user-agent": "Mushin.js knowledge base" }
|
|
444
|
+
});
|
|
445
|
+
} catch (error) {
|
|
446
|
+
throw new MushinError("unavailable", `Could not fetch ${url}: ${errorText(error)}`);
|
|
447
|
+
}
|
|
448
|
+
if (!response.ok) throw new MushinError("unavailable", `${url} answered HTTP ${response.status}`);
|
|
449
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
450
|
+
const type = response.headers.get("content-type") ?? "";
|
|
451
|
+
const format = type.includes("pdf") ? "pdf" : type.includes("html") || !type ? "html" : formatOf(new URL(url).pathname) ?? "txt";
|
|
452
|
+
return {
|
|
453
|
+
bytes,
|
|
454
|
+
format,
|
|
455
|
+
title: format === "html" ? htmlToText(new TextDecoder().decode(bytes)).title : void 0
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
async removeDoc(id) {
|
|
459
|
+
const doc = await this.doc(id);
|
|
460
|
+
await this.db.remove("knowledge_chunk", { doc: id });
|
|
461
|
+
await this.db.remove("knowledge_doc", { id });
|
|
462
|
+
await rm(this.filePath(doc), { force: true });
|
|
463
|
+
this.indexes.delete(doc.base);
|
|
464
|
+
this.changed(doc.base);
|
|
465
|
+
return { ok: true };
|
|
466
|
+
}
|
|
467
|
+
/** Parses a document again from its file (a web page is fetched again); edits to its chunks are lost. */
|
|
468
|
+
async reparse(id) {
|
|
469
|
+
const doc = await this.doc(id);
|
|
470
|
+
if (doc.kind === "notes" || doc.kind === "learned") throw new MushinError("invalid_input", "This document is made of single entries and has nothing to parse");
|
|
471
|
+
if (doc.kind === "url") {
|
|
472
|
+
const fetched = await this.fetchPage(doc.source);
|
|
473
|
+
await writeFile(this.filePath({
|
|
474
|
+
...doc,
|
|
475
|
+
format: fetched.format
|
|
476
|
+
}), fetched.bytes);
|
|
477
|
+
doc.format = fetched.format;
|
|
478
|
+
doc.hash = hashOf(fetched.bytes);
|
|
479
|
+
}
|
|
480
|
+
await this.db.upsert("knowledge_doc", [{
|
|
481
|
+
...doc,
|
|
482
|
+
status: "pending",
|
|
483
|
+
error: "",
|
|
484
|
+
updatedAt: Date.now()
|
|
485
|
+
}]);
|
|
486
|
+
this.changed(doc.base);
|
|
487
|
+
this.enqueue(() => this.process(id));
|
|
488
|
+
return { ok: true };
|
|
489
|
+
}
|
|
490
|
+
enqueue(task) {
|
|
491
|
+
this.queue = this.queue.catch(() => {}).then(() => this.abort.signal.aborted ? void 0 : task());
|
|
492
|
+
return this.queue;
|
|
493
|
+
}
|
|
494
|
+
async setStatus(doc, status, extra = {}) {
|
|
495
|
+
Object.assign(doc, {
|
|
496
|
+
status,
|
|
497
|
+
updatedAt: Date.now(),
|
|
498
|
+
...extra
|
|
499
|
+
});
|
|
500
|
+
await this.db.upsert("knowledge_doc", [doc]);
|
|
501
|
+
this.changed(doc.base);
|
|
502
|
+
}
|
|
503
|
+
/** Embeds texts with a base's model, `onBatch` after each batch; nothing for keyword-only bases. */
|
|
504
|
+
async embedTexts(base, texts, onBatch) {
|
|
505
|
+
if (!base.provider || !base.model || !texts.length) return void 0;
|
|
506
|
+
const vectors = [];
|
|
507
|
+
const BATCH = 32;
|
|
508
|
+
for (let index = 0; index < texts.length; index += BATCH) {
|
|
509
|
+
const result = await this.home.llm.embed({
|
|
510
|
+
provider: base.provider,
|
|
511
|
+
model: base.model,
|
|
512
|
+
texts: texts.slice(index, index + BATCH),
|
|
513
|
+
signal: this.abort.signal
|
|
514
|
+
});
|
|
515
|
+
vectors.push(...result.vectors.map(normalize));
|
|
516
|
+
onBatch?.(Math.min(index + BATCH, texts.length));
|
|
517
|
+
}
|
|
518
|
+
return vectors;
|
|
519
|
+
}
|
|
520
|
+
/** What a chunk's embedding reads: its title and text. */
|
|
521
|
+
embedText(chunk) {
|
|
522
|
+
return chunk.title ? `${chunk.title}\n${chunk.text}` : chunk.text;
|
|
523
|
+
}
|
|
524
|
+
async process(id) {
|
|
525
|
+
const [doc] = await this.db.get("knowledge_doc", { id });
|
|
526
|
+
if (!doc) return;
|
|
527
|
+
let base;
|
|
528
|
+
try {
|
|
529
|
+
base = await this.get(doc.base);
|
|
530
|
+
} catch {
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
try {
|
|
534
|
+
await this.setStatus(doc, "parsing");
|
|
535
|
+
this.progress({
|
|
536
|
+
base: base.id,
|
|
537
|
+
doc: id,
|
|
538
|
+
stage: "parsing",
|
|
539
|
+
done: 0,
|
|
540
|
+
total: 0
|
|
541
|
+
});
|
|
542
|
+
const bytes = new Uint8Array(await readFile(this.filePath(doc)));
|
|
543
|
+
const text = (await extractText(doc.format, bytes)).text.slice(0, MAX_TEXT_CHARS);
|
|
544
|
+
const pieces = chunkText(text, {
|
|
545
|
+
size: base.chunkSize,
|
|
546
|
+
overlap: base.chunkOverlap
|
|
547
|
+
}, doc.title);
|
|
548
|
+
if (!pieces.length) throw new MushinError("invalid_input", "No text found in this document");
|
|
549
|
+
await this.setStatus(doc, "embedding", { chars: text.length });
|
|
550
|
+
this.progress({
|
|
551
|
+
base: base.id,
|
|
552
|
+
doc: id,
|
|
553
|
+
stage: "embedding",
|
|
554
|
+
done: 0,
|
|
555
|
+
total: pieces.length
|
|
556
|
+
});
|
|
557
|
+
const vectors = await this.embedTexts(base, pieces.map((piece) => this.embedText(piece)), (done) => {
|
|
558
|
+
this.progress({
|
|
559
|
+
base: base.id,
|
|
560
|
+
doc: id,
|
|
561
|
+
stage: "embedding",
|
|
562
|
+
done,
|
|
563
|
+
total: pieces.length
|
|
564
|
+
});
|
|
565
|
+
});
|
|
566
|
+
const now = Date.now();
|
|
567
|
+
const rows = pieces.map((piece, seq) => ({
|
|
568
|
+
id: randomId(8),
|
|
569
|
+
base: base.id,
|
|
570
|
+
doc: id,
|
|
571
|
+
seq,
|
|
572
|
+
title: piece.title,
|
|
573
|
+
text: piece.text,
|
|
574
|
+
vector: vectorBytes(vectors?.[seq]),
|
|
575
|
+
edited: false,
|
|
576
|
+
meta: {},
|
|
577
|
+
createdAt: now,
|
|
578
|
+
updatedAt: now
|
|
579
|
+
}));
|
|
580
|
+
await this.db.remove("knowledge_chunk", { doc: id });
|
|
581
|
+
await this.db.upsert("knowledge_chunk", rows);
|
|
582
|
+
this.indexes.delete(base.id);
|
|
583
|
+
await this.setStatus(doc, "ready", {
|
|
584
|
+
chunks: rows.length,
|
|
585
|
+
error: ""
|
|
586
|
+
});
|
|
587
|
+
this.progress({
|
|
588
|
+
base: base.id,
|
|
589
|
+
doc: id,
|
|
590
|
+
stage: "done",
|
|
591
|
+
done: rows.length,
|
|
592
|
+
total: rows.length
|
|
593
|
+
});
|
|
594
|
+
this.logger.info("\"%s\" is ready: %d chunk(s)", doc.title, rows.length);
|
|
595
|
+
} catch (error) {
|
|
596
|
+
if (this.abort.signal.aborted) return;
|
|
597
|
+
this.logger.warn("reading \"%s\" failed: %s", doc.title, errorText(error));
|
|
598
|
+
await this.setStatus(doc, "error", { error: errorText(error) }).catch(() => {});
|
|
599
|
+
this.progress({
|
|
600
|
+
base: base.id,
|
|
601
|
+
doc: id,
|
|
602
|
+
stage: "error",
|
|
603
|
+
done: 0,
|
|
604
|
+
total: 0
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
/** Embeds every chunk of a base again with its current model, e.g. after the model changed. */
|
|
609
|
+
async reindex(id) {
|
|
610
|
+
const base = await this.get(id);
|
|
611
|
+
if (!base.provider) throw new MushinError("invalid_input", "This base has no embedding model; it searches by keywords only");
|
|
612
|
+
return this.enqueue(async () => {
|
|
613
|
+
const rows = await this.db.get("knowledge_chunk", { base: id });
|
|
614
|
+
this.progress({
|
|
615
|
+
base: id,
|
|
616
|
+
stage: "reindex",
|
|
617
|
+
done: 0,
|
|
618
|
+
total: rows.length
|
|
619
|
+
});
|
|
620
|
+
try {
|
|
621
|
+
const vectors = await this.embedTexts(base, rows.map((row) => this.embedText(row)), (done) => {
|
|
622
|
+
this.progress({
|
|
623
|
+
base: id,
|
|
624
|
+
stage: "reindex",
|
|
625
|
+
done,
|
|
626
|
+
total: rows.length
|
|
627
|
+
});
|
|
628
|
+
}) ?? [];
|
|
629
|
+
await this.db.upsert("knowledge_chunk", rows.map((row, index) => ({
|
|
630
|
+
...row,
|
|
631
|
+
vector: vectorBytes(vectors[index])
|
|
632
|
+
})));
|
|
633
|
+
this.logger.info("embedded %d chunk(s) of %s again", rows.length, base.id);
|
|
634
|
+
} catch (error) {
|
|
635
|
+
if (!this.abort.signal.aborted) this.logger.warn("embedding %s again failed: %s", base.id, errorText(error));
|
|
636
|
+
}
|
|
637
|
+
this.indexes.delete(id);
|
|
638
|
+
this.progress({
|
|
639
|
+
base: id,
|
|
640
|
+
stage: "done",
|
|
641
|
+
done: rows.length,
|
|
642
|
+
total: rows.length
|
|
643
|
+
});
|
|
644
|
+
this.changed(id);
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
async chunks(input) {
|
|
648
|
+
await this.get(input.base);
|
|
649
|
+
const rows = await this.db.get("knowledge_chunk", input.doc ? {
|
|
650
|
+
base: input.base,
|
|
651
|
+
doc: input.doc
|
|
652
|
+
} : { base: input.base }, { sort: {
|
|
653
|
+
doc: "asc",
|
|
654
|
+
seq: "asc"
|
|
655
|
+
} });
|
|
656
|
+
const query = input.query?.trim().toLowerCase();
|
|
657
|
+
const matched = query ? rows.filter((row) => row.text.toLowerCase().includes(query) || row.title.toLowerCase().includes(query)) : rows;
|
|
658
|
+
const offset = input.offset ?? 0;
|
|
659
|
+
return {
|
|
660
|
+
total: matched.length,
|
|
661
|
+
chunks: matched.slice(offset, offset + (input.limit ?? 50)).map(publicChunk)
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
async chunkRow(id) {
|
|
665
|
+
const [row] = await this.db.get("knowledge_chunk", { id });
|
|
666
|
+
if (!row) throw new MushinError("not_found", `Chunk "${id}" not found`);
|
|
667
|
+
return row;
|
|
668
|
+
}
|
|
669
|
+
async updateChunk(id, patch) {
|
|
670
|
+
const row = await this.chunkRow(id);
|
|
671
|
+
const base = await this.get(row.base);
|
|
672
|
+
const next = {
|
|
673
|
+
...row,
|
|
674
|
+
...patch.title !== void 0 ? { title: patch.title } : {},
|
|
675
|
+
...patch.text !== void 0 ? { text: patch.text } : {},
|
|
676
|
+
edited: true,
|
|
677
|
+
updatedAt: Date.now()
|
|
678
|
+
};
|
|
679
|
+
if (!next.text.trim()) throw new MushinError("invalid_input", "A chunk needs text; remove it instead");
|
|
680
|
+
const [vector] = await this.embedTexts(base, [this.embedText(next)]) ?? [];
|
|
681
|
+
next.vector = vectorBytes(vector);
|
|
682
|
+
await this.db.upsert("knowledge_chunk", [next]);
|
|
683
|
+
this.indexes.delete(row.base);
|
|
684
|
+
this.changed(row.base);
|
|
685
|
+
return publicChunk(next);
|
|
686
|
+
}
|
|
687
|
+
async removeChunk(id) {
|
|
688
|
+
const row = await this.chunkRow(id);
|
|
689
|
+
await this.db.remove("knowledge_chunk", { id });
|
|
690
|
+
await this.recount(row.doc);
|
|
691
|
+
this.indexes.delete(row.base);
|
|
692
|
+
this.changed(row.base);
|
|
693
|
+
return { ok: true };
|
|
694
|
+
}
|
|
695
|
+
/** Joins a chunk with the one after it in the same document. */
|
|
696
|
+
async mergeChunk(id) {
|
|
697
|
+
const row = await this.chunkRow(id);
|
|
698
|
+
const [after] = await this.db.get("knowledge_chunk", {
|
|
699
|
+
doc: row.doc,
|
|
700
|
+
seq: { $gt: row.seq }
|
|
701
|
+
}, {
|
|
702
|
+
sort: { seq: "asc" },
|
|
703
|
+
limit: 1
|
|
704
|
+
});
|
|
705
|
+
if (!after) throw new MushinError("invalid_input", "This is the last chunk of its document");
|
|
706
|
+
await this.db.remove("knowledge_chunk", { id: after.id });
|
|
707
|
+
const merged = await this.updateChunk(id, { text: `${row.text}\n\n${after.text}` });
|
|
708
|
+
await this.recount(row.doc);
|
|
709
|
+
return merged;
|
|
710
|
+
}
|
|
711
|
+
async recount(docId) {
|
|
712
|
+
const [doc] = await this.db.get("knowledge_doc", { id: docId });
|
|
713
|
+
if (!doc) return;
|
|
714
|
+
const count = (await this.db.get("knowledge_chunk", { doc: docId }, ["id"])).length;
|
|
715
|
+
await this.db.upsert("knowledge_doc", [{
|
|
716
|
+
...doc,
|
|
717
|
+
chunks: count,
|
|
718
|
+
updatedAt: Date.now()
|
|
719
|
+
}]);
|
|
720
|
+
}
|
|
721
|
+
/** The document of single entries of a kind, made the first time one is added. */
|
|
722
|
+
async entriesDoc(base, kind) {
|
|
723
|
+
const [found] = await this.db.get("knowledge_doc", {
|
|
724
|
+
base: base.id,
|
|
725
|
+
kind
|
|
726
|
+
});
|
|
727
|
+
if (found) return found;
|
|
728
|
+
const now = Date.now();
|
|
729
|
+
const doc = {
|
|
730
|
+
id: randomId(6),
|
|
731
|
+
base: base.id,
|
|
732
|
+
title: kind === "notes" ? NOTES_TITLE : LEARNED_TITLE,
|
|
733
|
+
kind,
|
|
734
|
+
source: "",
|
|
735
|
+
format: "md",
|
|
736
|
+
status: "ready",
|
|
737
|
+
error: "",
|
|
738
|
+
hash: `${kind}:${base.id}`,
|
|
739
|
+
chars: 0,
|
|
740
|
+
chunks: 0,
|
|
741
|
+
createdAt: now,
|
|
742
|
+
updatedAt: now
|
|
743
|
+
};
|
|
744
|
+
await this.db.upsert("knowledge_doc", [doc]);
|
|
745
|
+
return doc;
|
|
746
|
+
}
|
|
747
|
+
/** Adds one entry, such as a `/记住` note or an accepted candidate, as a chunk of its own. */
|
|
748
|
+
async addEntry(baseId, entry) {
|
|
749
|
+
const base = await this.get(baseId);
|
|
750
|
+
const text = entry.text.trim();
|
|
751
|
+
if (!text) throw new MushinError("invalid_input", "The entry is empty");
|
|
752
|
+
const doc = await this.entriesDoc(base, entry.kind ?? "notes");
|
|
753
|
+
const [last] = await this.db.get("knowledge_chunk", { doc: doc.id }, {
|
|
754
|
+
sort: { seq: "desc" },
|
|
755
|
+
limit: 1
|
|
756
|
+
});
|
|
757
|
+
const title = entry.title?.trim() ?? "";
|
|
758
|
+
const [vector] = await this.embedTexts(base, [this.embedText({
|
|
759
|
+
title,
|
|
760
|
+
text
|
|
761
|
+
})]) ?? [];
|
|
762
|
+
const now = Date.now();
|
|
763
|
+
const row = {
|
|
764
|
+
id: randomId(8),
|
|
765
|
+
base: base.id,
|
|
766
|
+
doc: doc.id,
|
|
767
|
+
seq: (last?.seq ?? -1) + 1,
|
|
768
|
+
title,
|
|
769
|
+
text,
|
|
770
|
+
vector: vectorBytes(vector),
|
|
771
|
+
edited: false,
|
|
772
|
+
meta: entry.meta ?? {},
|
|
773
|
+
createdAt: now,
|
|
774
|
+
updatedAt: now
|
|
775
|
+
};
|
|
776
|
+
await this.db.upsert("knowledge_chunk", [row]);
|
|
777
|
+
await this.recount(doc.id);
|
|
778
|
+
this.indexes.delete(base.id);
|
|
779
|
+
this.changed(base.id);
|
|
780
|
+
return publicChunk(row);
|
|
781
|
+
}
|
|
782
|
+
/** Removes notes that contain the text; returns how many. */
|
|
783
|
+
async forget(baseId, text) {
|
|
784
|
+
const wanted = text.trim().toLowerCase();
|
|
785
|
+
if (!wanted) return 0;
|
|
786
|
+
const docs = await this.db.get("knowledge_doc", {
|
|
787
|
+
base: baseId,
|
|
788
|
+
kind: "notes"
|
|
789
|
+
});
|
|
790
|
+
let removed = 0;
|
|
791
|
+
for (const doc of docs) {
|
|
792
|
+
const matching = (await this.db.get("knowledge_chunk", { doc: doc.id })).filter((row) => row.text.toLowerCase().includes(wanted));
|
|
793
|
+
if (!matching.length) continue;
|
|
794
|
+
await this.db.remove("knowledge_chunk", { id: matching.map((row) => row.id) });
|
|
795
|
+
await this.recount(doc.id);
|
|
796
|
+
removed += matching.length;
|
|
797
|
+
}
|
|
798
|
+
if (removed) {
|
|
799
|
+
this.indexes.delete(baseId);
|
|
800
|
+
this.changed(baseId);
|
|
801
|
+
}
|
|
802
|
+
return removed;
|
|
803
|
+
}
|
|
804
|
+
async index(base) {
|
|
805
|
+
let index = this.indexes.get(base);
|
|
806
|
+
if (index) return index;
|
|
807
|
+
const [record] = await this.db.get("knowledge_base", { id: base });
|
|
808
|
+
const rows = await this.db.get("knowledge_chunk", { base });
|
|
809
|
+
index = new SearchIndex(rows.map((row) => {
|
|
810
|
+
const vector = vectorOf(row.vector);
|
|
811
|
+
return {
|
|
812
|
+
id: row.id,
|
|
813
|
+
doc: row.doc,
|
|
814
|
+
title: row.title,
|
|
815
|
+
text: row.text,
|
|
816
|
+
...vector && record?.dimension === vector.length ? { vector } : {}
|
|
817
|
+
};
|
|
818
|
+
}));
|
|
819
|
+
this.indexes.set(base, index);
|
|
820
|
+
return index;
|
|
821
|
+
}
|
|
822
|
+
/**
|
|
823
|
+
* Searches bases (every one when none are given) by keywords and, where the base has an
|
|
824
|
+
* embedding model, by meaning. A failing embedding model falls back to keywords with a warning.
|
|
825
|
+
*/
|
|
826
|
+
async search(input) {
|
|
827
|
+
const all = await this.list();
|
|
828
|
+
const bases = input.bases?.length ? input.bases.map((id) => {
|
|
829
|
+
const found = all.find((base) => base.id === id || base.name === id);
|
|
830
|
+
if (!found) throw new MushinError("not_found", `Knowledge base "${id}" not found`);
|
|
831
|
+
return found;
|
|
832
|
+
}) : all;
|
|
833
|
+
const query = input.query.trim();
|
|
834
|
+
const warnings = [];
|
|
835
|
+
if (!query || !bases.length) return {
|
|
836
|
+
hits: [],
|
|
837
|
+
semantic: false,
|
|
838
|
+
warnings
|
|
839
|
+
};
|
|
840
|
+
const vectors = /* @__PURE__ */ new Map();
|
|
841
|
+
let semantic = false;
|
|
842
|
+
const hits = [];
|
|
843
|
+
const docTitles = /* @__PURE__ */ new Map();
|
|
844
|
+
for (const base of bases) {
|
|
845
|
+
let vector;
|
|
846
|
+
if (base.provider && base.model && base.dimension) {
|
|
847
|
+
const key = `${base.provider}|${base.model}`;
|
|
848
|
+
if (!vectors.has(key)) try {
|
|
849
|
+
const result = await this.home.llm.embed({
|
|
850
|
+
provider: base.provider,
|
|
851
|
+
model: base.model,
|
|
852
|
+
texts: [query],
|
|
853
|
+
signal: input.signal ?? AbortSignal.timeout(2e4)
|
|
854
|
+
});
|
|
855
|
+
vectors.set(key, normalize(result.vectors[0]));
|
|
856
|
+
} catch (error) {
|
|
857
|
+
if (input.signal?.aborted) throw error;
|
|
858
|
+
vectors.set(key, void 0);
|
|
859
|
+
warnings.push(`${base.name}: the embedding model failed (${errorText(error)}); searched by keywords only`);
|
|
860
|
+
}
|
|
861
|
+
vector = vectors.get(key);
|
|
862
|
+
if (vector) semantic = true;
|
|
863
|
+
}
|
|
864
|
+
const index = await this.index(base.id);
|
|
865
|
+
for (const hit of index.search(query, vector, {
|
|
866
|
+
topK: input.topK ?? base.topK,
|
|
867
|
+
minScore: input.minScore ?? base.minScore
|
|
868
|
+
})) {
|
|
869
|
+
if (!docTitles.has(hit.doc)) docTitles.set(hit.doc, (await this.db.get("knowledge_doc", { id: hit.doc }, ["title"]))[0]?.title ?? "");
|
|
870
|
+
hits.push({
|
|
871
|
+
...hit,
|
|
872
|
+
base: base.id,
|
|
873
|
+
baseName: base.name,
|
|
874
|
+
docTitle: docTitles.get(hit.doc)
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
const topK = input.topK ?? Math.max(...bases.map((base) => base.topK));
|
|
879
|
+
hits.sort((a, b) => b.fused - a.fused || b.score - a.score);
|
|
880
|
+
return {
|
|
881
|
+
hits: hits.slice(0, topK),
|
|
882
|
+
semantic,
|
|
883
|
+
warnings
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
/** Chunks and links between similar ones, for the similarity map. */
|
|
887
|
+
async graph(input) {
|
|
888
|
+
const base = await this.get(input.base);
|
|
889
|
+
const index = await this.index(base.id);
|
|
890
|
+
const { nodes, edges, byVector } = index.links(input);
|
|
891
|
+
const docs = new Map((await this.db.get("knowledge_doc", { base: base.id })).map((doc) => [doc.id, doc.title]));
|
|
892
|
+
return {
|
|
893
|
+
byVector,
|
|
894
|
+
total: index.size,
|
|
895
|
+
nodes: nodes.map((chunk) => ({
|
|
896
|
+
id: chunk.id,
|
|
897
|
+
doc: chunk.doc,
|
|
898
|
+
docTitle: docs.get(chunk.doc) ?? "",
|
|
899
|
+
title: chunk.title,
|
|
900
|
+
text: chunk.text.slice(0, 300)
|
|
901
|
+
})),
|
|
902
|
+
edges
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
/** Flows that ask an LLM, for "use in a flow", with whether they already search this base. */
|
|
906
|
+
flowChoices(base) {
|
|
907
|
+
const flows = this.home.get("flows");
|
|
908
|
+
if (!flows) return [];
|
|
909
|
+
return flows.list().flatMap((summary) => {
|
|
910
|
+
const doc = flows.get(summary.id);
|
|
911
|
+
if (!doc.nodes.some((node) => node.type === "ai.chat")) return [];
|
|
912
|
+
const search = doc.nodes.find((node) => node.type === "kb.search");
|
|
913
|
+
return [{
|
|
914
|
+
id: doc.id,
|
|
915
|
+
name: doc.name,
|
|
916
|
+
bots: doc.bots,
|
|
917
|
+
llmChat: doc.source?.kind === "plugin" && doc.source.plugin === "@mushinjs/bot-plugin-llm-chat",
|
|
918
|
+
uses: !!search && (!base || (search.config.bases ?? []).includes(base))
|
|
919
|
+
}];
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
/**
|
|
923
|
+
* Puts a knowledge search in front of a flow's first "Ask the LLM" node and passes its passages
|
|
924
|
+
* to it; a flow that already has a search node gets these bases added to it.
|
|
925
|
+
*/
|
|
926
|
+
async useInFlow(flowId, bases) {
|
|
927
|
+
const flows = this.home.get("flows");
|
|
928
|
+
if (!flows) throw new MushinError("unavailable", "Flows are not running");
|
|
929
|
+
for (const id of bases) await this.get(id);
|
|
930
|
+
const doc = flows.get(flowId);
|
|
931
|
+
const existing = doc.nodes.find((node) => node.type === "kb.search");
|
|
932
|
+
if (existing) {
|
|
933
|
+
const merged = [.../* @__PURE__ */ new Set([...existing.config.bases ?? [], ...bases])];
|
|
934
|
+
const nodes = doc.nodes.map((node) => node === existing ? {
|
|
935
|
+
...node,
|
|
936
|
+
config: {
|
|
937
|
+
...node.config,
|
|
938
|
+
bases: merged
|
|
939
|
+
}
|
|
940
|
+
} : node);
|
|
941
|
+
await flows.update(flowId, { nodes });
|
|
942
|
+
return {
|
|
943
|
+
flow: flowId,
|
|
944
|
+
node: existing.id,
|
|
945
|
+
added: false
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
const chat = doc.nodes.find((node) => node.type === "ai.chat");
|
|
949
|
+
if (!chat) throw new MushinError("invalid_input", `"${doc.name}" has no "Ask the LLM" node`);
|
|
950
|
+
let id = "kb";
|
|
951
|
+
for (let n = 2; doc.nodes.some((node) => node.id === id); n++) id = `kb${n}`;
|
|
952
|
+
const shift = 280;
|
|
953
|
+
const search = {
|
|
954
|
+
id,
|
|
955
|
+
type: "kb.search",
|
|
956
|
+
label: "知识库检索",
|
|
957
|
+
position: { ...chat.position },
|
|
958
|
+
config: {
|
|
959
|
+
bases,
|
|
960
|
+
topK: 3,
|
|
961
|
+
minScore: .3
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
const nodes = [...doc.nodes.map((node) => {
|
|
965
|
+
const moved = node.position.x >= chat.position.x ? {
|
|
966
|
+
...node,
|
|
967
|
+
position: {
|
|
968
|
+
x: node.position.x + shift,
|
|
969
|
+
y: node.position.y
|
|
970
|
+
}
|
|
971
|
+
} : node;
|
|
972
|
+
return node === chat ? {
|
|
973
|
+
...moved,
|
|
974
|
+
inputs: {
|
|
975
|
+
...chat.inputs,
|
|
976
|
+
knowledge: `{{${id}.context}}`
|
|
977
|
+
}
|
|
978
|
+
} : moved;
|
|
979
|
+
}), search];
|
|
980
|
+
const edges = [
|
|
981
|
+
...doc.edges.map((edge) => edge.target === chat.id && edge.targetHandle === "exec" ? {
|
|
982
|
+
...edge,
|
|
983
|
+
target: id
|
|
984
|
+
} : edge),
|
|
985
|
+
{
|
|
986
|
+
source: id,
|
|
987
|
+
sourceHandle: "exec:next",
|
|
988
|
+
target: chat.id,
|
|
989
|
+
targetHandle: "exec"
|
|
990
|
+
},
|
|
991
|
+
{
|
|
992
|
+
source: id,
|
|
993
|
+
sourceHandle: "exec:empty",
|
|
994
|
+
target: chat.id,
|
|
995
|
+
targetHandle: "exec"
|
|
996
|
+
}
|
|
997
|
+
];
|
|
998
|
+
await flows.update(flowId, {
|
|
999
|
+
nodes,
|
|
1000
|
+
edges
|
|
1001
|
+
});
|
|
1002
|
+
this.logger.info("flow %s now searches knowledge before asking the LLM", doc.id);
|
|
1003
|
+
return {
|
|
1004
|
+
flow: flowId,
|
|
1005
|
+
node: id,
|
|
1006
|
+
added: true
|
|
1007
|
+
};
|
|
1008
|
+
}
|
|
1009
|
+
/** Bases a bot may add to from chat, oldest first. */
|
|
1010
|
+
async basesFor(botId) {
|
|
1011
|
+
return (await this.list()).filter((base) => base.commands.enabled && base.bots.includes(botId));
|
|
1012
|
+
}
|
|
1013
|
+
*defineCommands() {
|
|
1014
|
+
const bots = this.home.bots;
|
|
1015
|
+
const hasBases = /* @__PURE__ */ new Set();
|
|
1016
|
+
const refresh = async () => {
|
|
1017
|
+
hasBases.clear();
|
|
1018
|
+
for (const base of await this.list()) if (base.commands.enabled) for (const bot of base.bots) hasBases.add(bot);
|
|
1019
|
+
};
|
|
1020
|
+
refresh();
|
|
1021
|
+
yield this.home.on("knowledge/changed", () => void refresh());
|
|
1022
|
+
const only = { bots: (botId) => hasBases.has(botId) };
|
|
1023
|
+
yield bots.defineCommand({
|
|
1024
|
+
name: "记住",
|
|
1025
|
+
aliases: ["remember"],
|
|
1026
|
+
description: "把一句话记进知识库",
|
|
1027
|
+
args: [{
|
|
1028
|
+
name: "内容",
|
|
1029
|
+
type: "text",
|
|
1030
|
+
rest: true
|
|
1031
|
+
}]
|
|
1032
|
+
}, async (session) => {
|
|
1033
|
+
const base = await this.allowed(session);
|
|
1034
|
+
if (!base) return;
|
|
1035
|
+
const text = session.command?.rest.trim() ?? "";
|
|
1036
|
+
if (!text) return session.send("用法:/记住 要记住的内容");
|
|
1037
|
+
await this.addEntry(base.id, {
|
|
1038
|
+
text,
|
|
1039
|
+
meta: this.metaOf(session)
|
|
1040
|
+
});
|
|
1041
|
+
await session.send(`已记进知识库「${base.name}」`);
|
|
1042
|
+
}, only);
|
|
1043
|
+
yield bots.defineCommand({
|
|
1044
|
+
name: "收录",
|
|
1045
|
+
aliases: ["collect"],
|
|
1046
|
+
description: "回复一条消息,把它收进知识库"
|
|
1047
|
+
}, async (session) => {
|
|
1048
|
+
const base = await this.allowed(session);
|
|
1049
|
+
if (!base) return;
|
|
1050
|
+
const quoted = session.segments.find((segment) => segment.type === "reply")?.data?.id;
|
|
1051
|
+
if (!quoted) return session.send("请回复要收录的那条消息,再发 /收录");
|
|
1052
|
+
const message = (await bots.messages(session.bot.id, {
|
|
1053
|
+
channel: session.channel.id,
|
|
1054
|
+
limit: 500
|
|
1055
|
+
})).find((record) => record.messageId === String(quoted));
|
|
1056
|
+
const text = message ? message.segments.filter((segment) => segment.type === "text").map((segment) => segment.data.text).join("").trim() : "";
|
|
1057
|
+
if (!text) return session.send("没找到那条消息的文字内容(可能太早,或者不是文字消息)");
|
|
1058
|
+
await this.addEntry(base.id, {
|
|
1059
|
+
text,
|
|
1060
|
+
meta: {
|
|
1061
|
+
...this.metaOf(session),
|
|
1062
|
+
from: message.user.id,
|
|
1063
|
+
fromName: message.user.name ?? ""
|
|
1064
|
+
}
|
|
1065
|
+
});
|
|
1066
|
+
await session.send(`已收录到知识库「${base.name}」`);
|
|
1067
|
+
}, only);
|
|
1068
|
+
yield bots.defineCommand({
|
|
1069
|
+
name: "忘记",
|
|
1070
|
+
aliases: ["forget"],
|
|
1071
|
+
description: "从知识库删除记住的内容",
|
|
1072
|
+
args: [{
|
|
1073
|
+
name: "内容",
|
|
1074
|
+
type: "text",
|
|
1075
|
+
rest: true
|
|
1076
|
+
}]
|
|
1077
|
+
}, async (session) => {
|
|
1078
|
+
const base = await this.allowed(session);
|
|
1079
|
+
if (!base) return;
|
|
1080
|
+
const text = session.command?.rest.trim() ?? "";
|
|
1081
|
+
if (!text) return session.send("用法:/忘记 要删除的内容(会删除包含这段文字的记录)");
|
|
1082
|
+
const removed = await this.forget(base.id, text);
|
|
1083
|
+
await session.send(removed ? `已从知识库「${base.name}」删除 ${removed} 条` : `知识库「${base.name}」里没有用 /记住 或 /收录 记下的、包含“${text}”的内容`);
|
|
1084
|
+
}, only);
|
|
1085
|
+
}
|
|
1086
|
+
metaOf(session) {
|
|
1087
|
+
return {
|
|
1088
|
+
bot: session.bot.id,
|
|
1089
|
+
channel: `${session.channel.type}:${session.channel.id}`,
|
|
1090
|
+
user: session.user.id,
|
|
1091
|
+
userName: session.user.name ?? ""
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
/** The base a chat command writes to, after checking the sender may use it; tells them when not. */
|
|
1095
|
+
async allowed(session) {
|
|
1096
|
+
const [base] = await this.basesFor(session.bot.id);
|
|
1097
|
+
if (!base) return;
|
|
1098
|
+
if (!roleAtLeast(session.role() ?? "user", base.commands.role)) {
|
|
1099
|
+
await session.send(base.commands.role === "master" ? "只有主人可以修改知识库" : "需要管理员及以上才能修改知识库");
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
return base;
|
|
1103
|
+
}
|
|
1104
|
+
onMessage(record) {
|
|
1105
|
+
if (!record.segments.some((segment) => segment.type === "text")) return;
|
|
1106
|
+
this.list().then((bases) => {
|
|
1107
|
+
const chat = `${record.channel.type}:${record.channel.id}`;
|
|
1108
|
+
for (const base of bases) {
|
|
1109
|
+
const learn = {
|
|
1110
|
+
...DEFAULT_LEARN,
|
|
1111
|
+
...base.learn
|
|
1112
|
+
};
|
|
1113
|
+
if (!learn.enabled || !base.bots.includes(record.botId)) continue;
|
|
1114
|
+
if (learn.channels.length && !learn.channels.includes(chat)) continue;
|
|
1115
|
+
const key = `${base.id}|${record.botId}|${chat}`;
|
|
1116
|
+
const count = (this.counters.get(key) ?? 0) + 1;
|
|
1117
|
+
if (count < learn.every) {
|
|
1118
|
+
this.counters.set(key, count);
|
|
1119
|
+
continue;
|
|
1120
|
+
}
|
|
1121
|
+
this.counters.delete(key);
|
|
1122
|
+
this.learn(base.id, record.botId, chat).catch((error) => this.logger.warn("learning from %s failed: %s", chat, errorText(error)));
|
|
1123
|
+
}
|
|
1124
|
+
}, () => {});
|
|
1125
|
+
}
|
|
1126
|
+
async useQuota(base) {
|
|
1127
|
+
const today = localDay();
|
|
1128
|
+
if (this.usage.day !== today) this.usage = {
|
|
1129
|
+
day: today,
|
|
1130
|
+
counts: {}
|
|
1131
|
+
};
|
|
1132
|
+
const used = this.usage.counts[base.id] ?? 0;
|
|
1133
|
+
if (used >= base.learn.dailyLimit) return false;
|
|
1134
|
+
this.usage.counts[base.id] = used + 1;
|
|
1135
|
+
await this.home.storage.set(USAGE_KEY, this.usage);
|
|
1136
|
+
return true;
|
|
1137
|
+
}
|
|
1138
|
+
/**
|
|
1139
|
+
* Reads a chat's messages since it was last learned from and asks the LLM for knowledge worth
|
|
1140
|
+
* keeping. What it finds waits for review, or is added straight away with auto-accept.
|
|
1141
|
+
*/
|
|
1142
|
+
async learn(baseId, botId, chat) {
|
|
1143
|
+
const base = await this.get(baseId);
|
|
1144
|
+
const learn = {
|
|
1145
|
+
...DEFAULT_LEARN,
|
|
1146
|
+
...base.learn
|
|
1147
|
+
};
|
|
1148
|
+
const key = `${baseId}|${botId}|${chat}`;
|
|
1149
|
+
if (this.learning.has(key)) return {
|
|
1150
|
+
candidates: [],
|
|
1151
|
+
skipped: "already learning from this chat"
|
|
1152
|
+
};
|
|
1153
|
+
const bot = this.home.bots.get(botId);
|
|
1154
|
+
const [type, ...rest] = chat.split(":");
|
|
1155
|
+
const channelId = rest.join(":");
|
|
1156
|
+
const since = this.learnState[key] ?? 0;
|
|
1157
|
+
const lines = (await this.home.bots.messages(botId, {
|
|
1158
|
+
channel: channelId,
|
|
1159
|
+
limit: Math.max(learn.every, 50)
|
|
1160
|
+
})).filter((record) => record.channel.type === type && record.timestamp > since);
|
|
1161
|
+
const text = lines.map((record) => ({
|
|
1162
|
+
direction: record.direction,
|
|
1163
|
+
user: record.user,
|
|
1164
|
+
text: record.segments.filter((segment) => segment.type === "text").map((segment) => segment.data.text).join("")
|
|
1165
|
+
}));
|
|
1166
|
+
if (text.filter((line) => line.text.trim()).length < LEARN_MIN_LINES) return {
|
|
1167
|
+
candidates: [],
|
|
1168
|
+
skipped: "not enough new messages"
|
|
1169
|
+
};
|
|
1170
|
+
if (!await this.useQuota(base)) return {
|
|
1171
|
+
candidates: [],
|
|
1172
|
+
skipped: "the daily learning limit is used up"
|
|
1173
|
+
};
|
|
1174
|
+
this.learning.add(key);
|
|
1175
|
+
try {
|
|
1176
|
+
const result = await this.home.llm.chat({
|
|
1177
|
+
provider: learn.provider || bot.provider || void 0,
|
|
1178
|
+
model: learn.model || void 0,
|
|
1179
|
+
maxTokens: 2e3,
|
|
1180
|
+
signal: AbortSignal.any([this.abort.signal, AbortSignal.timeout(9e4)]),
|
|
1181
|
+
messages: [{
|
|
1182
|
+
role: "system",
|
|
1183
|
+
content: LEARN_PROMPT
|
|
1184
|
+
}, {
|
|
1185
|
+
role: "user",
|
|
1186
|
+
content: transcript(text, bot.name)
|
|
1187
|
+
}]
|
|
1188
|
+
});
|
|
1189
|
+
this.learnState[key] = Math.max(...lines.map((record) => record.timestamp));
|
|
1190
|
+
await this.home.storage.set(LEARN_STATE_KEY, this.learnState);
|
|
1191
|
+
const found = await this.newCandidates(base, parseCandidates(result.text));
|
|
1192
|
+
const now = Date.now();
|
|
1193
|
+
const created = [];
|
|
1194
|
+
for (const candidate of found) {
|
|
1195
|
+
const row = await this.db.create("knowledge_candidate", {
|
|
1196
|
+
base: baseId,
|
|
1197
|
+
bot: botId,
|
|
1198
|
+
channel: chat,
|
|
1199
|
+
question: candidate.question,
|
|
1200
|
+
answer: candidate.answer,
|
|
1201
|
+
status: "pending",
|
|
1202
|
+
chunk: "",
|
|
1203
|
+
createdAt: now,
|
|
1204
|
+
updatedAt: now
|
|
1205
|
+
});
|
|
1206
|
+
created.push(learn.autoAccept ? await this.accept(row.id) : row);
|
|
1207
|
+
}
|
|
1208
|
+
this.logger.info("learned %d item(s) for \"%s\" from %s", created.length, base.name, chat);
|
|
1209
|
+
if (created.length) this.changed(baseId);
|
|
1210
|
+
return { candidates: created };
|
|
1211
|
+
} finally {
|
|
1212
|
+
this.learning.delete(key);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
/** Leaves out candidates already waiting for review or already known to the base. */
|
|
1216
|
+
async newCandidates(base, found) {
|
|
1217
|
+
const pending = new Set((await this.db.get("knowledge_candidate", {
|
|
1218
|
+
base: base.id,
|
|
1219
|
+
status: "pending"
|
|
1220
|
+
}, ["question"])).map((row) => row.question));
|
|
1221
|
+
const fresh = [];
|
|
1222
|
+
for (const candidate of found) {
|
|
1223
|
+
if (pending.has(candidate.question)) continue;
|
|
1224
|
+
const { hits } = await this.search({
|
|
1225
|
+
bases: [base.id],
|
|
1226
|
+
query: candidate.question,
|
|
1227
|
+
topK: 1,
|
|
1228
|
+
minScore: .9
|
|
1229
|
+
}).catch(() => ({ hits: [] }));
|
|
1230
|
+
if (hits.length) continue;
|
|
1231
|
+
fresh.push(candidate);
|
|
1232
|
+
}
|
|
1233
|
+
return fresh;
|
|
1234
|
+
}
|
|
1235
|
+
async candidates(base, status) {
|
|
1236
|
+
await this.get(base);
|
|
1237
|
+
return this.db.get("knowledge_candidate", status ? {
|
|
1238
|
+
base,
|
|
1239
|
+
status
|
|
1240
|
+
} : { base }, { sort: { createdAt: "desc" } });
|
|
1241
|
+
}
|
|
1242
|
+
async candidate(id) {
|
|
1243
|
+
const [row] = await this.db.get("knowledge_candidate", { id });
|
|
1244
|
+
if (!row) throw new MushinError("not_found", `Candidate ${id} not found`);
|
|
1245
|
+
return row;
|
|
1246
|
+
}
|
|
1247
|
+
/** Adds a candidate (as edited) to the base's "learned from chats" document. */
|
|
1248
|
+
async accept(id, patch = {}) {
|
|
1249
|
+
const row = await this.candidate(id);
|
|
1250
|
+
if (row.status === "accepted") return row;
|
|
1251
|
+
const question = patch.question?.trim() || row.question;
|
|
1252
|
+
const answer = patch.answer?.trim() || row.answer;
|
|
1253
|
+
const chunk = await this.addEntry(row.base, {
|
|
1254
|
+
kind: "learned",
|
|
1255
|
+
title: question,
|
|
1256
|
+
text: candidateText({
|
|
1257
|
+
question,
|
|
1258
|
+
answer
|
|
1259
|
+
}),
|
|
1260
|
+
meta: {
|
|
1261
|
+
bot: row.bot,
|
|
1262
|
+
channel: row.channel,
|
|
1263
|
+
candidate: row.id
|
|
1264
|
+
}
|
|
1265
|
+
});
|
|
1266
|
+
const next = {
|
|
1267
|
+
...row,
|
|
1268
|
+
question,
|
|
1269
|
+
answer,
|
|
1270
|
+
status: "accepted",
|
|
1271
|
+
chunk: chunk.id,
|
|
1272
|
+
updatedAt: Date.now()
|
|
1273
|
+
};
|
|
1274
|
+
await this.db.upsert("knowledge_candidate", [next]);
|
|
1275
|
+
this.changed(row.base);
|
|
1276
|
+
return next;
|
|
1277
|
+
}
|
|
1278
|
+
async reject(id) {
|
|
1279
|
+
const row = await this.candidate(id);
|
|
1280
|
+
const next = {
|
|
1281
|
+
...row,
|
|
1282
|
+
status: "rejected",
|
|
1283
|
+
updatedAt: Date.now()
|
|
1284
|
+
};
|
|
1285
|
+
await this.db.upsert("knowledge_candidate", [next]);
|
|
1286
|
+
this.changed(row.base);
|
|
1287
|
+
return next;
|
|
1288
|
+
}
|
|
1289
|
+
*defineActions() {
|
|
1290
|
+
const actions = this.home.actions;
|
|
1291
|
+
const baseId = z.object({ base: z.string().describe("Knowledge base id from kb.list") });
|
|
1292
|
+
yield actions.define({
|
|
1293
|
+
name: "kb.list",
|
|
1294
|
+
description: "List knowledge bases: name, embedding model (none means keyword search only), document, chunk and pending candidate counts, linked bots and learning settings.",
|
|
1295
|
+
risk: "read",
|
|
1296
|
+
input: z.object({}),
|
|
1297
|
+
handler: () => this.summaries()
|
|
1298
|
+
});
|
|
1299
|
+
yield actions.define({
|
|
1300
|
+
name: "kb.choices",
|
|
1301
|
+
description: "Knowledge base ids and names, for pickers.",
|
|
1302
|
+
risk: "read",
|
|
1303
|
+
input: z.object({}),
|
|
1304
|
+
handler: async () => (await this.list()).map((base) => ({
|
|
1305
|
+
value: base.id,
|
|
1306
|
+
label: base.name
|
|
1307
|
+
}))
|
|
1308
|
+
});
|
|
1309
|
+
yield actions.define({
|
|
1310
|
+
name: "kb.create",
|
|
1311
|
+
description: "Create a knowledge base. With an embedding provider and model (providers.embedChoices) it searches by meaning as well as keywords; the model is tried once. Link bots to let their admins add knowledge from chat (/记住, /收录, /忘记) and to learn from their chats.",
|
|
1312
|
+
risk: "write",
|
|
1313
|
+
input: z.object(baseFields),
|
|
1314
|
+
handler: (input) => this.create(input)
|
|
1315
|
+
});
|
|
1316
|
+
yield actions.define({
|
|
1317
|
+
name: "kb.update",
|
|
1318
|
+
description: "Change a knowledge base. Changing the embedding model embeds every chunk again in the background. Lists given replace the old ones.",
|
|
1319
|
+
risk: "write",
|
|
1320
|
+
input: z.object({ id: z.string() }).extend(z.object(baseFields).partial().shape),
|
|
1321
|
+
handler: ({ id, ...patch }) => this.update(id, patch)
|
|
1322
|
+
});
|
|
1323
|
+
yield actions.define({
|
|
1324
|
+
name: "kb.delete",
|
|
1325
|
+
description: "Delete a knowledge base with its documents, chunks and candidates.",
|
|
1326
|
+
risk: "dangerous",
|
|
1327
|
+
input: z.object({ id: z.string() }),
|
|
1328
|
+
handler: ({ id }) => this.remove(id)
|
|
1329
|
+
});
|
|
1330
|
+
yield actions.define({
|
|
1331
|
+
name: "kb.docs.list",
|
|
1332
|
+
description: "Documents of a knowledge base with their status (pending, parsing, embedding, ready, error), size and chunk count.",
|
|
1333
|
+
risk: "read",
|
|
1334
|
+
input: baseId,
|
|
1335
|
+
handler: ({ base }) => this.docs(base)
|
|
1336
|
+
});
|
|
1337
|
+
yield actions.define({
|
|
1338
|
+
name: "kb.docs.add",
|
|
1339
|
+
description: "Add a document to a knowledge base: pasted text (Markdown works best: headings become chunk titles), a web page url, or a file as base64 data with its name (md, txt, html; pdf and docx need optional packages). It is split and embedded in the background.",
|
|
1340
|
+
risk: "write",
|
|
1341
|
+
input: baseId.extend({
|
|
1342
|
+
name: z.string().optional().describe("Title, or the file name for data"),
|
|
1343
|
+
text: z.string().optional(),
|
|
1344
|
+
url: z.string().optional(),
|
|
1345
|
+
data: z.string().optional().describe("File content in base64")
|
|
1346
|
+
}),
|
|
1347
|
+
handler: (input) => this.addDoc(input)
|
|
1348
|
+
});
|
|
1349
|
+
yield actions.define({
|
|
1350
|
+
name: "kb.docs.remove",
|
|
1351
|
+
description: "Remove a document and its chunks from its knowledge base.",
|
|
1352
|
+
risk: "dangerous",
|
|
1353
|
+
input: z.object({ id: z.string().describe("Document id") }),
|
|
1354
|
+
handler: ({ id }) => this.removeDoc(id)
|
|
1355
|
+
});
|
|
1356
|
+
yield actions.define({
|
|
1357
|
+
name: "kb.docs.reparse",
|
|
1358
|
+
description: "Split a document again from its file (a web page is fetched again). Edits to its chunks are lost.",
|
|
1359
|
+
risk: "write",
|
|
1360
|
+
input: z.object({ id: z.string() }),
|
|
1361
|
+
handler: ({ id }) => this.reparse(id)
|
|
1362
|
+
});
|
|
1363
|
+
yield actions.define({
|
|
1364
|
+
name: "kb.chunks.list",
|
|
1365
|
+
description: "Chunks of a knowledge base (or one document), in order, optionally only those containing some text.",
|
|
1366
|
+
risk: "read",
|
|
1367
|
+
input: baseId.extend({
|
|
1368
|
+
doc: z.string().optional(),
|
|
1369
|
+
query: z.string().optional().describe("Only chunks containing this text"),
|
|
1370
|
+
offset: z.number().int().min(0).optional(),
|
|
1371
|
+
limit: z.number().int().min(1).max(200).optional()
|
|
1372
|
+
}),
|
|
1373
|
+
handler: (input) => this.chunks(input)
|
|
1374
|
+
});
|
|
1375
|
+
yield actions.define({
|
|
1376
|
+
name: "kb.chunks.add",
|
|
1377
|
+
description: "Add one entry to a knowledge base (it goes in its \"随手记\" document), as /记住 does in chat.",
|
|
1378
|
+
risk: "write",
|
|
1379
|
+
input: baseId.extend({
|
|
1380
|
+
text: z.string().min(1),
|
|
1381
|
+
title: z.string().optional()
|
|
1382
|
+
}),
|
|
1383
|
+
handler: ({ base, text, title }) => this.addEntry(base, {
|
|
1384
|
+
text,
|
|
1385
|
+
title,
|
|
1386
|
+
meta: { by: "dashboard" }
|
|
1387
|
+
})
|
|
1388
|
+
});
|
|
1389
|
+
yield actions.define({
|
|
1390
|
+
name: "kb.chunks.update",
|
|
1391
|
+
description: "Change a chunk's title or text; it is embedded again.",
|
|
1392
|
+
risk: "write",
|
|
1393
|
+
input: z.object({
|
|
1394
|
+
id: z.string(),
|
|
1395
|
+
title: z.string().optional(),
|
|
1396
|
+
text: z.string().optional()
|
|
1397
|
+
}),
|
|
1398
|
+
handler: ({ id, ...patch }) => this.updateChunk(id, patch)
|
|
1399
|
+
});
|
|
1400
|
+
yield actions.define({
|
|
1401
|
+
name: "kb.chunks.remove",
|
|
1402
|
+
description: "Remove one chunk.",
|
|
1403
|
+
risk: "write",
|
|
1404
|
+
input: z.object({ id: z.string() }),
|
|
1405
|
+
handler: ({ id }) => this.removeChunk(id)
|
|
1406
|
+
});
|
|
1407
|
+
yield actions.define({
|
|
1408
|
+
name: "kb.chunks.merge",
|
|
1409
|
+
description: "Join a chunk with the next one of its document.",
|
|
1410
|
+
risk: "write",
|
|
1411
|
+
input: z.object({ id: z.string() }),
|
|
1412
|
+
handler: ({ id }) => this.mergeChunk(id)
|
|
1413
|
+
});
|
|
1414
|
+
yield actions.define({
|
|
1415
|
+
name: "kb.search",
|
|
1416
|
+
description: "Search knowledge bases (all when none are given) for passages about a question. Each hit has its text, title, document, a score from 0 to 1 (the better of meaning similarity and the share of the question's words it contains) and its ranks by keywords and by meaning. Answer from the hits and cite them; say so when nothing relevant is found.",
|
|
1417
|
+
risk: "read",
|
|
1418
|
+
input: z.object({
|
|
1419
|
+
query: z.string().min(1),
|
|
1420
|
+
bases: z.array(z.string()).optional().describe("Base ids or names"),
|
|
1421
|
+
topK: z.number().int().min(1).max(20).optional(),
|
|
1422
|
+
minScore: z.number().min(0).max(1).optional()
|
|
1423
|
+
}),
|
|
1424
|
+
handler: (input) => this.search(input)
|
|
1425
|
+
});
|
|
1426
|
+
yield actions.define({
|
|
1427
|
+
name: "kb.reindex",
|
|
1428
|
+
description: "Embed every chunk of a knowledge base again with its model, in the background.",
|
|
1429
|
+
risk: "write",
|
|
1430
|
+
input: z.object({ id: z.string() }),
|
|
1431
|
+
handler: async ({ id }) => {
|
|
1432
|
+
this.reindex(id);
|
|
1433
|
+
return { ok: true };
|
|
1434
|
+
}
|
|
1435
|
+
});
|
|
1436
|
+
yield actions.define({
|
|
1437
|
+
name: "kb.graph",
|
|
1438
|
+
description: "Chunks of a knowledge base and links between similar ones (by embeddings, or shared words without them), for the similarity map.",
|
|
1439
|
+
risk: "read",
|
|
1440
|
+
input: baseId.extend({
|
|
1441
|
+
neighbors: z.number().int().min(1).max(10).optional(),
|
|
1442
|
+
minSimilarity: z.number().min(0).max(1).optional(),
|
|
1443
|
+
limit: z.number().int().min(10).max(2e3).optional()
|
|
1444
|
+
}),
|
|
1445
|
+
handler: (input) => this.graph(input)
|
|
1446
|
+
});
|
|
1447
|
+
yield actions.define({
|
|
1448
|
+
name: "kb.flowChoices",
|
|
1449
|
+
description: "Flows with an \"Ask the LLM\" node, which kb.useInFlow can add a knowledge search to; says which already search the base.",
|
|
1450
|
+
risk: "read",
|
|
1451
|
+
input: z.object({ base: z.string().optional() }),
|
|
1452
|
+
handler: ({ base }) => this.flowChoices(base)
|
|
1453
|
+
});
|
|
1454
|
+
yield actions.define({
|
|
1455
|
+
name: "kb.useInFlow",
|
|
1456
|
+
description: "Make a flow search knowledge bases before its first \"Ask the LLM\" node and give it the passages found (a bot's LLM chat flow, for example). A flow that already has a knowledge search node gets the bases added.",
|
|
1457
|
+
risk: "write",
|
|
1458
|
+
input: z.object({
|
|
1459
|
+
flow: z.string(),
|
|
1460
|
+
bases: z.array(z.string()).min(1)
|
|
1461
|
+
}),
|
|
1462
|
+
handler: ({ flow, bases }) => this.useInFlow(flow, bases)
|
|
1463
|
+
});
|
|
1464
|
+
yield actions.define({
|
|
1465
|
+
name: "kb.candidates.list",
|
|
1466
|
+
description: "Knowledge learned from chats, newest first: pending ones wait for review.",
|
|
1467
|
+
risk: "read",
|
|
1468
|
+
input: baseId.extend({ status: z.enum([
|
|
1469
|
+
"pending",
|
|
1470
|
+
"accepted",
|
|
1471
|
+
"rejected"
|
|
1472
|
+
]).optional() }),
|
|
1473
|
+
handler: ({ base, status }) => this.candidates(base, status)
|
|
1474
|
+
});
|
|
1475
|
+
yield actions.define({
|
|
1476
|
+
name: "kb.candidates.accept",
|
|
1477
|
+
description: "Add a learned candidate to its knowledge base, optionally with an edited question or answer.",
|
|
1478
|
+
risk: "write",
|
|
1479
|
+
input: z.object({
|
|
1480
|
+
id: z.number().int(),
|
|
1481
|
+
question: z.string().optional(),
|
|
1482
|
+
answer: z.string().optional()
|
|
1483
|
+
}),
|
|
1484
|
+
handler: ({ id, ...patch }) => this.accept(id, patch)
|
|
1485
|
+
});
|
|
1486
|
+
yield actions.define({
|
|
1487
|
+
name: "kb.candidates.reject",
|
|
1488
|
+
description: "Ignore a learned candidate.",
|
|
1489
|
+
risk: "write",
|
|
1490
|
+
input: z.object({ id: z.number().int() }),
|
|
1491
|
+
handler: ({ id }) => this.reject(id)
|
|
1492
|
+
});
|
|
1493
|
+
yield actions.define({
|
|
1494
|
+
name: "kb.learn",
|
|
1495
|
+
description: "Learn from one chat now instead of waiting for enough messages: reads its messages since the last time and asks the LLM for knowledge worth keeping. Costs one LLM request.",
|
|
1496
|
+
risk: "write",
|
|
1497
|
+
input: baseId.extend({
|
|
1498
|
+
bot: z.string(),
|
|
1499
|
+
chat: z.string().regex(/^(group|private):\S+$/).describe("\"group:<id>\" or \"private:<user id>\"")
|
|
1500
|
+
}),
|
|
1501
|
+
handler: ({ base, bot, chat }) => this.learn(base, bot, chat)
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
};
|
|
1505
|
+
export { FORMATS, Knowledge, Knowledge as default, LEARN_PROMPT, SearchIndex, candidateText, chunkText, dot, extractText, formatOf, htmlToText, normalize, parseCandidates, passages, tokenize, transcript };
|
|
1506
|
+
|
|
1507
|
+
//# sourceMappingURL=index.js.map
|