dsh-redteam-memory 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +219 -0
- package/cordis.patch.yml +10 -0
- package/lib/client.js +918 -0
- package/lib/host.js +1828 -0
- package/lib/parts/client.head.js +15 -0
- package/lib/parts/client.shim.js +45 -0
- package/lib/parts/client.tail.js +5 -0
- package/lib/parts/host.head.js +80 -0
- package/lib/parts/host.tail.js +37 -0
- package/package.json +70 -0
- package/src/client.js +860 -0
- package/src/host.js +1711 -0
- package/tools/build-lib.mjs +86 -0
- package/tools/prepare-gh-packages.mjs +66 -0
package/src/host.js
ADDED
|
@@ -0,0 +1,1711 @@
|
|
|
1
|
+
// 红队记忆 · Host 半边主体
|
|
2
|
+
//
|
|
3
|
+
// 本文件是 applyHost 的【函数体】——函数头、harness 垫片、收尾与导出都由
|
|
4
|
+
// lib/parts/host.head.js 与 host.tail.js 提供,所以这里不要写 import、function 头或 return 块。
|
|
5
|
+
// lib/host.js 由 `npm run build:lib` 生成,不要手改 lib/。
|
|
6
|
+
//
|
|
7
|
+
// ── 这个插件干什么 ────────────────────────────────────────────────────────────
|
|
8
|
+
// 给模型一个**可检索的 AI 安全知识库**:把 AI 安全知识与进攻技巧(OWASP LLM Top 10、
|
|
9
|
+
// 提示词注入、RAG 投毒、Agent/MCP 工具滥用、推理服务未授权与已知 CVE…)存进本地库,
|
|
10
|
+
// 模型用 memory_search 检索,人可以在面板上导入 / 删除 / 浏览。
|
|
11
|
+
//
|
|
12
|
+
// ── 架构:本地库是权威数据,Milvus 只是它的派生索引 ────────────────────────────
|
|
13
|
+
// .redteam-memory.json(本地条目,一定写得进去)
|
|
14
|
+
// └─ 同步 → Milvus(向量召回 + 可选重排:jina / siliconflow / dashscope / cohere / bigmodel)
|
|
15
|
+
// 向量模型(阿里 tongyi-embedding-vision-flash / 智谱 Embedding-3 / OpenAI 兼容)
|
|
16
|
+
// └─ 同步失败也不影响读写:检索自动退回本地关键词匹配,面板上能看到待同步条数。
|
|
17
|
+
// MinIO(Milvus 用的存储桶,例如 mimo):连通测试 / 列举 / 整库导出(导出源同样是本地库)
|
|
18
|
+
//
|
|
19
|
+
// 这条「本地优先」的顺序是刻意的:记忆是给人用的,不该因为外部服务没配好就写不进去。
|
|
20
|
+
// 对话捕获(工作区里有人说「写入记忆」)尤其依赖它 —— 捕获必须「说了就记住」。
|
|
21
|
+
//
|
|
22
|
+
// ── 为什么 HTTP 走 shell+curl 而不是 fetch ────────────────────────────────────
|
|
23
|
+
// 动态半边屏蔽了 fetch(见 ../../docs/DEVELOPMENT.md),而本插件两种形态都要能跑。
|
|
24
|
+
// 资产图谱调 Jina MCP 用的就是同一套:ctx.get('shell') → curl。顺带一个好处:
|
|
25
|
+
// curl 自带 --aws-sigv4,S3 的签名不用手写。
|
|
26
|
+
//
|
|
27
|
+
// 动态半边写文件的沙箱边界(实测,见 ../../docs/DEVELOPMENT.md §5.1):
|
|
28
|
+
// fs.resolve(相对路径) 落在**插件自己的默认工作区**(本机是 /home/kali/桌面),
|
|
29
|
+
// 写它之外的绝对路径会被拒(file access denied under workspace-write mode),
|
|
30
|
+
// 但**读**任何路径都允许。所以 storePath 默认用相对名,并把解析出的宿主路径显示在面板上。
|
|
31
|
+
//
|
|
32
|
+
// 静态形态与动态半边的差异(详见 ../../docs/DEVELOPMENT.md):
|
|
33
|
+
// - 工具与 RPC 统一用 harness.defineTool / harness.registerTool / harness.handle
|
|
34
|
+
// (静态包的 host.head.js 提供垫片把它们映射到 defineTool / ctx.tools.register / HTTP 路由)
|
|
35
|
+
|
|
36
|
+
// ── 常量 ──────────────────────────────────────────────────────────────────
|
|
37
|
+
const STORE_NAME = '.redteam-memory.json'
|
|
38
|
+
const STORE_VERSION = 2
|
|
39
|
+
const LOG_MAX = 120
|
|
40
|
+
const ENTRY_MAX = 5000
|
|
41
|
+
const CAPTURE_MAX = 50
|
|
42
|
+
const INDEX_BATCH = 16
|
|
43
|
+
|
|
44
|
+
// 导入只认这四类。刻意不扩:格式越多,「导入失败」的成因就越多,而这四类覆盖了
|
|
45
|
+
// 红队现场真正会拿到的东西(报告 / 笔记 / 扫描导出的 txt)。解析都不引依赖:
|
|
46
|
+
// md、txt 直接读;docx 借 unzip 取 word/document.xml;pdf 借 pdftotext(poppler)。
|
|
47
|
+
const IMPORT_EXT = ['.md', '.markdown', '.txt', '.docx', '.pdf']
|
|
48
|
+
const IMPORT_LABEL = 'pdf / word(.docx) / md / txt'
|
|
49
|
+
// word / pdf 抽出来的是纯文本、没有 markdown 记号,按空行分段再按长度归拢,
|
|
50
|
+
// 否则整份文档会变成一条巨大条目(切块后标题只剩「x(1/40)」)。
|
|
51
|
+
const PLAIN_GROUP_MAX = 2000
|
|
52
|
+
|
|
53
|
+
// 对话捕获的默认触发词。命中任意一个就把这条消息收进记忆库 ——
|
|
54
|
+
// 「写入记忆」这类说法是红队队员的自然表达,不该要求他们换用工具。
|
|
55
|
+
const CAPTURE_PHRASES = '写入记忆,写到记忆,记到记忆,记入记忆,存到记忆,存入记忆,存进记忆,记录到记忆,加入记忆,加到记忆,记住这点,记住这个,remember this,save to memory'
|
|
56
|
+
const HTTP_TIMEOUT_MS = 60000
|
|
57
|
+
const EMBED_BATCH = 16
|
|
58
|
+
const INSERT_BATCH = 50
|
|
59
|
+
const CHUNK_MAX = 1200
|
|
60
|
+
const CHUNK_OVERLAP = 120
|
|
61
|
+
const LIST_DEFAULT = 50
|
|
62
|
+
const KINDS = ['knowledge', 'technique', 'payload', 'checklist', 'note']
|
|
63
|
+
|
|
64
|
+
// 向量模型预设。dimension 是默认维度,建集合时用它;换模型必须换集合或删掉重建。
|
|
65
|
+
const EMBED_PRESETS = {
|
|
66
|
+
dashscope: {
|
|
67
|
+
label: '阿里百炼 DashScope',
|
|
68
|
+
base: 'https://dashscope.aliyuncs.com',
|
|
69
|
+
keyHint: 'sk-…(百炼控制台的 API Key)',
|
|
70
|
+
models: [
|
|
71
|
+
{ id: 'tongyi-embedding-vision-flash', dimension: 1024, note: '多模态(文本+图)flash,默认 1024 维' },
|
|
72
|
+
{ id: 'tongyi-embedding-vision-plus', dimension: 1152, note: '多模态 plus' },
|
|
73
|
+
{ id: 'text-embedding-v4', dimension: 1024, note: '纯文本,可选 1024/768/512/256/128/64' },
|
|
74
|
+
],
|
|
75
|
+
},
|
|
76
|
+
bigmodel: {
|
|
77
|
+
label: '智谱 BigModel',
|
|
78
|
+
base: 'https://open.bigmodel.cn',
|
|
79
|
+
keyHint: '…(bigmodel.cn 的 API Key)',
|
|
80
|
+
models: [
|
|
81
|
+
{ id: 'embedding-3', dimension: 2048, note: '默认 2048 维,可指定 2048/1024/512/256' },
|
|
82
|
+
{ id: 'embedding-2', dimension: 1024, note: '上一代,1024 维' },
|
|
83
|
+
],
|
|
84
|
+
},
|
|
85
|
+
openai: {
|
|
86
|
+
label: 'OpenAI 兼容(自建 / 其它厂商)',
|
|
87
|
+
base: '',
|
|
88
|
+
keyHint: '按你的服务填,不需要鉴权可留空',
|
|
89
|
+
models: [
|
|
90
|
+
{ id: 'text-embedding-3-large', dimension: 3072, note: 'OpenAI' },
|
|
91
|
+
{ id: 'text-embedding-3-small', dimension: 1536, note: 'OpenAI' },
|
|
92
|
+
{ id: 'bge-m3', dimension: 1024, note: '本地 vLLM / Ollama 常见' },
|
|
93
|
+
],
|
|
94
|
+
},
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// 重排模型预设:常见的都列上,直接选,不用记路径。
|
|
98
|
+
const RERANK_PRESETS = {
|
|
99
|
+
jina: { label: 'Jina Reranker', base: 'https://api.jina.ai', path: '/v1/rerank', models: ['jina-reranker-v2-base-multilingual', 'jina-reranker-v3', 'jina-reranker-m0'] },
|
|
100
|
+
siliconflow: { label: 'SiliconFlow 硅基流动', base: 'https://api.siliconflow.cn', path: '/v1/rerank', models: ['BAAI/bge-reranker-v2-m3', 'BAAI/bge-reranker-v2-minicpm-layerwise', 'Qwen/Qwen3-Reranker-8B', 'netease-youdao/bce-reranker-base_v1'] },
|
|
101
|
+
dashscope: { label: '阿里百炼 gte-rerank', base: 'https://dashscope.aliyuncs.com', path: '/api/v1/services/rerank/text-rerank/text-rerank', models: ['gte-rerank-v2', 'qwen3-reranker-8b'] },
|
|
102
|
+
cohere: { label: 'Cohere Rerank', base: 'https://api.cohere.com', path: '/v1/rerank', models: ['rerank-v3.5', 'rerank-multilingual-v3.0', 'rerank-english-v3.0'] },
|
|
103
|
+
bigmodel: { label: '智谱 Rerank', base: 'https://open.bigmodel.cn', path: '/api/paas/v4/rerank', models: ['rerank'] },
|
|
104
|
+
custom: { label: '自定义(OpenAI 兼容 /v1/rerank)', base: '', path: '/v1/rerank', models: [] },
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// 内置知识包:装完点一下「导入内置知识包」就能检索,不必先自己灌数据。
|
|
108
|
+
// 每条都是公开、可写进报告的知识点,不是可直接执行的东西。
|
|
109
|
+
const SEED_KNOWLEDGE = [
|
|
110
|
+
{ title: 'OWASP LLM01 提示词注入', kind: 'knowledge', tags: 'owasp,注入', text: '把指令伪装成数据塞进模型上下文,使它偏离原定职责。直接注入 = 用户在对话里覆盖系统指令;间接注入 = 载荷藏进模型会读到的文档/网页/工具返回里。测试要点:先确认哪些输入真的进了模型上下文 —— 只读 message 字段的系统,往其它字段塞载荷是无效的;再确认工具返回值是否被当指令执行。' },
|
|
111
|
+
{ title: 'OWASP LLM02 敏感信息泄露', kind: 'knowledge', tags: 'owasp,泄露', text: '模型输出里带出不该带出的内容:系统提示词、内部文档、他人数据、凭据。要区分「模型复述了上下文里的东西」与「应用把敏感数据拼进了上下文」,后者才是可用发现。留证时同时保存原始请求与响应。' },
|
|
112
|
+
{ title: 'OWASP LLM03 供应链', kind: 'knowledge', tags: 'owasp,供应链', text: '模型权重、微调数据、推理框架、依赖包任一环被污染都会传导到应用。红队视角:确认推理服务与版本(例如 Ollama 的版本号)、模型来源与哈希、是否允许外部拉取模型;公开 CVE 与官方披露是主要线索来源。' },
|
|
113
|
+
{ title: 'OWASP LLM04 数据与模型投毒', kind: 'knowledge', tags: 'owasp,投毒', text: '污染训练或检索数据以影响模型行为。RAG 场景更现实的是投毒检索库:只要能写进知识库,就可能影响后续回答。验证关键是「写进去的内容会不会被读出来」—— 写入计数增长不等于内容被处理,必须找到读回路径才算数。' },
|
|
114
|
+
{ title: 'OWASP LLM05 不当输出处理', kind: 'knowledge', tags: 'owasp,下游注入', text: '模型输出被下游直接当代码/命令/HTML/SQL 使用。关注输出是否流入 shell、模板、SQL、浏览器 DOM(XSS)或工具参数。这条通常与「Agent 工具调用」串起来打。' },
|
|
115
|
+
{ title: 'OWASP LLM06 过度代理权限', kind: 'knowledge', tags: 'owasp,agent', text: 'Agent 拿着超出需要的权限(能删数据、能发外部请求、能读文件)。测试要点:逐一确认每个工具的真实副作用,而不是看它的描述 —— 描述写着只读的工具也可能真的会改数据。' },
|
|
116
|
+
{ title: 'OWASP LLM07 系统提示词泄露', kind: 'knowledge', tags: 'owasp,提示词', text: '套出系统提示词。常见手法:要求复述、角色扮演、翻译/编码变换、要求以 JSON 或表格输出上下文。拿到后重点看有没有藏在提示词里的凭据、内部端点、业务规则。' },
|
|
117
|
+
{ title: 'OWASP LLM08 向量与嵌入弱点', kind: 'knowledge', tags: 'owasp,向量库', text: '嵌入与向量库本身可被攻击:越权读写 collection、检索越权(搜到别的租户内容)、嵌入反演(从向量近似还原原文)。红队要确认向量库是否有认证、是否按租户隔离、是否暴露管理接口。' },
|
|
118
|
+
{ title: 'OWASP LLM09 错误信息(幻觉)', kind: 'knowledge', tags: 'owasp,幻觉', text: '模型编造不存在的事实并让使用者据此行动。红队价值在于证明「编造内容会被下游当真」—— 例如让 Agent 依据幻觉去调用工具或写数据。' },
|
|
119
|
+
{ title: 'OWASP LLM10 模型拒绝服务', kind: 'knowledge', tags: 'owasp,dos', text: '用超长上下文、递归工具调用、批量并发把推理资源吃满。测试限定在授权环境且要能恢复:先记录 QPS/耗时基线,再对比攻击后的基线,用数据说明影响。' },
|
|
120
|
+
{ title: '间接注入:文档上传链路', kind: 'technique', tags: '注入,rag', text: '把载荷写成「正常文档」(季度报告、README、工单),诱导模型在检索时读到。验证链看三段:上传是否真的落库(计数/列表)、文档是否被处理进上下文(检索命中)、载荷是否被执行(工具被调用或内容被反射)。三段缺一段就不算成功。' },
|
|
121
|
+
{ title: '工具与 MCP 服务枚举', kind: 'technique', tags: '枚举,agent', text: '先问出模型有哪些工具/命令("你有哪些可以调用的工具或函数?"),再逐个验证是否真的存在、是否真的有副作用。很多目标的工具列表是装饰性的 —— 只在提示词里写着,后端没有对应端点。' },
|
|
122
|
+
{ title: 'RAG 索引目标收集', kind: 'technique', tags: 'rag,收集', text: '知识库往往比对话更值钱。路径:确认知识库接口(上传/列表/读取)、确认读写是否同一份数据、用主题词把内容勾出来(按 topic 注入再按 topic 查询)。写入路径与读取路径不相通是常见现象。' },
|
|
123
|
+
{ title: '未授权访问与默认口令', kind: 'technique', tags: '未授权,凭据', text: '暴露的推理服务(Ollama / vLLM / TGI 等)常常没有认证。顺序:直接调用 API 看是否要凭据 → 试官方默认端口与默认口令 → 看响应头与错误信息是否泄露版本(版本决定可用 CVE)。所有验证限定在授权范围内。' },
|
|
124
|
+
{ title: '模型管理接口的滥用面', kind: 'technique', tags: 'ollama,cve', text: '推理服务的模型管理接口(/api/create、/api/pull、/api/delete)常被忽略:未授权时能创建/删除模型,部分版本还能借 from / modelfile 参数读取路径(GGUF 相关 CVE)。要点:先确认接口是否要鉴权,再看参数接受的输入类型,最后验证副作用(模型清单变化)。' },
|
|
125
|
+
{ title: '证据与合规:报告要留什么', kind: 'checklist', tags: '报告,证据', text: '每条确认的发现都要能复现:原始请求(含认证状态)、原始响应、时间戳、影响说明。测试结束要清理自己写入的东西(上传的文档、创建的模型、数据库里的过程),并在报告里写明清理结果 —— 这既是职业要求,也避免污染授权环境。' },
|
|
126
|
+
]
|
|
127
|
+
|
|
128
|
+
function blankSettings() {
|
|
129
|
+
return {
|
|
130
|
+
milvus: { uri: '', token: '', dbName: 'default', collection: 'redteam_memory', metric: 'COSINE' },
|
|
131
|
+
embed: { provider: 'dashscope', model: 'tongyi-embedding-vision-flash', apiKey: '', dimension: 1024, baseUrl: '' },
|
|
132
|
+
rerank: { enabled: false, provider: 'jina', model: 'jina-reranker-v2-base-multilingual', apiKey: '', baseUrl: '', topN: 8 },
|
|
133
|
+
s3: { enabled: false, endpoint: '', region: 'us-east-1', bucket: 'mimo', prefix: '', accessKey: '', secretKey: '', pathStyle: true },
|
|
134
|
+
// 对话捕获:工作区里有人说「写入记忆」就自动入库,不需要额外配置向量库。
|
|
135
|
+
capture: { enabled: true, phrases: CAPTURE_PHRASES, kind: 'note', withContext: true },
|
|
136
|
+
storePath: STORE_NAME,
|
|
137
|
+
searchTopK: 8,
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function blankStore() {
|
|
142
|
+
return {
|
|
143
|
+
version: STORE_VERSION,
|
|
144
|
+
updatedAt: 0,
|
|
145
|
+
settings: blankSettings(),
|
|
146
|
+
// 本地条目是**权威数据**:Milvus 只是它的派生索引。
|
|
147
|
+
// 这样没有向量模型 Key 也能记录与检索,网络故障也不会丢数据。
|
|
148
|
+
entries: [],
|
|
149
|
+
captures: [],
|
|
150
|
+
log: [],
|
|
151
|
+
logSeq: 0,
|
|
152
|
+
meta: {
|
|
153
|
+
collectionReady: false, dimension: 0, rowCount: 0, indexed: 0,
|
|
154
|
+
lastError: null, lastOp: null, progress: null, seeded: false,
|
|
155
|
+
persistence: 'unknown', storePath: '',
|
|
156
|
+
},
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ── 工具函数 ──────────────────────────────────────────────────────────────
|
|
161
|
+
function msgOf(e) { return e && e.message ? String(e.message) : String(e) }
|
|
162
|
+
function nowMs() { return Date.now() }
|
|
163
|
+
function clip(s, n) {
|
|
164
|
+
const v = String(s === undefined || s === null ? '' : s).replace(/\s+/g, ' ').trim()
|
|
165
|
+
return v.length > n ? v.slice(0, n - 1) + '…' : v
|
|
166
|
+
}
|
|
167
|
+
function intOf(v, d) { const n = Number(v); return Number.isFinite(n) ? Math.round(n) : d }
|
|
168
|
+
function boolOf(v, d) { return typeof v === 'boolean' ? v : (v === 'true' ? true : (v === 'false' ? false : d)) }
|
|
169
|
+
|
|
170
|
+
// 单引号包裹的 shell 参数。把 JSON 内联进命令时必须走它,否则引号会被 shell 吃掉。
|
|
171
|
+
function shQuote(s) { return "'" + String(s === undefined || s === null ? '' : s).replace(/'/g, "'\\''") + "'" }
|
|
172
|
+
|
|
173
|
+
// 稳定内容 id:动态沙箱里没有 crypto,双累加器 FNV 变体足够做去重。
|
|
174
|
+
function contentId(parts) {
|
|
175
|
+
const s = parts.join('\u0000')
|
|
176
|
+
let a = 0x811c9dc5, b = 0x01000193
|
|
177
|
+
for (let i = 0; i < s.length; i++) {
|
|
178
|
+
const c = s.charCodeAt(i)
|
|
179
|
+
a = (a ^ c) >>> 0; a = (a * 16777619) >>> 0
|
|
180
|
+
b = (b + c * (i + 1)) >>> 0; b = (b ^ (b << 5)) >>> 0
|
|
181
|
+
}
|
|
182
|
+
return 'm' + a.toString(16).padStart(8, '0') + b.toString(16).padStart(8, '0')
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function log(level, text) {
|
|
186
|
+
store.logSeq = (store.logSeq || 0) + 1
|
|
187
|
+
store.log.push({ seq: store.logSeq, at: nowMs(), level: level, text: String(text).slice(0, 1200) })
|
|
188
|
+
if (store.log.length > LOG_MAX) store.log = store.log.slice(store.log.length - LOG_MAX)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function logFailure(prefix, e, extra) {
|
|
192
|
+
log('err', prefix + ':' + msgOf(e) + (extra ? ' [' + extra + ']' : ''))
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// ── 落盘 ──────────────────────────────────────────────────────────────────
|
|
196
|
+
const store = blankStore()
|
|
197
|
+
let loaded = false
|
|
198
|
+
|
|
199
|
+
function settings() { return store.settings }
|
|
200
|
+
|
|
201
|
+
async function resolveTarget() {
|
|
202
|
+
const fs = ctx.get('fs')
|
|
203
|
+
if (fs === undefined || fs === null) return null
|
|
204
|
+
const target = await fs.resolve(store.settings.storePath || STORE_NAME)
|
|
205
|
+
return { fs: fs, target: target }
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// 只接受已知字段,避免界面上的脏数据写进设置。
|
|
209
|
+
function mergeSettings(src) {
|
|
210
|
+
const d = blankSettings()
|
|
211
|
+
const s = store.settings
|
|
212
|
+
const pick = function (group, keys) {
|
|
213
|
+
const g = src[group]
|
|
214
|
+
if (!g || typeof g !== 'object') return
|
|
215
|
+
for (const k of keys) if (g[k] !== undefined && g[k] !== null) s[group][k] = g[k]
|
|
216
|
+
}
|
|
217
|
+
pick('milvus', Object.keys(d.milvus))
|
|
218
|
+
pick('embed', Object.keys(d.embed))
|
|
219
|
+
pick('rerank', Object.keys(d.rerank))
|
|
220
|
+
pick('s3', Object.keys(d.s3))
|
|
221
|
+
pick('capture', ['enabled', 'kind', 'withContext'])
|
|
222
|
+
if (src.capture && src.capture.phrases !== undefined) s.capture.phrases = String(src.capture.phrases || '').slice(0, 2000)
|
|
223
|
+
if (src.storePath !== undefined) s.storePath = String(src.storePath || STORE_NAME)
|
|
224
|
+
if (src.searchTopK !== undefined) s.searchTopK = intOf(src.searchTopK, 8)
|
|
225
|
+
s.milvus.metric = /^(COSINE|L2|IP)$/.test(String(s.milvus.metric)) ? String(s.milvus.metric) : 'COSINE'
|
|
226
|
+
s.embed.dimension = intOf(s.embed.dimension, 1024)
|
|
227
|
+
s.rerank.topN = Math.max(1, Math.min(50, intOf(s.rerank.topN, 8)))
|
|
228
|
+
s.searchTopK = Math.max(1, Math.min(50, intOf(s.searchTopK, 8)))
|
|
229
|
+
s.rerank.enabled = boolOf(s.rerank.enabled, false)
|
|
230
|
+
s.s3.enabled = boolOf(s.s3.enabled, false)
|
|
231
|
+
s.s3.pathStyle = boolOf(s.s3.pathStyle, true)
|
|
232
|
+
s.capture.enabled = boolOf(s.capture.enabled, true)
|
|
233
|
+
s.capture.withContext = boolOf(s.capture.withContext, true)
|
|
234
|
+
if (KINDS.indexOf(String(s.capture.kind)) < 0) s.capture.kind = 'note'
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// 触发词表:逗号/顿号/换行分隔,去重、去空。
|
|
238
|
+
function capturePhrases(s) {
|
|
239
|
+
const raw = String((s || settings()).capture.phrases || '')
|
|
240
|
+
const out = []
|
|
241
|
+
for (const p of raw.split(/[,,、\n]/)) {
|
|
242
|
+
const v = p.trim()
|
|
243
|
+
if (v && out.indexOf(v) < 0) out.push(v)
|
|
244
|
+
}
|
|
245
|
+
return out
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function doLoad() {
|
|
249
|
+
try {
|
|
250
|
+
const r = await resolveTarget()
|
|
251
|
+
if (!r) { store.meta.persistence = 'memory'; log('warn', 'fs 服务不可用,本次仅内存保存'); return 0 }
|
|
252
|
+
store.meta.storePath = hostPathOf(r.fs, r.target)
|
|
253
|
+
const info = await r.fs.stat(r.target)
|
|
254
|
+
const parsed = info ? JSON.parse(await r.fs.readText(r.target)) : null
|
|
255
|
+
if (parsed && typeof parsed === 'object') {
|
|
256
|
+
// 版本只用于提示:设置与条目都按字段合并,升级不该丢掉已经配好的 Key。
|
|
257
|
+
if (Number(parsed.version) !== STORE_VERSION) {
|
|
258
|
+
log('warn', '存储版本 ' + parsed.version + ' -> ' + STORE_VERSION + ',按字段合并(不会丢设置)')
|
|
259
|
+
}
|
|
260
|
+
if (parsed.settings && typeof parsed.settings === 'object') mergeSettings(parsed.settings)
|
|
261
|
+
if (Array.isArray(parsed.entries)) {
|
|
262
|
+
for (const e of parsed.entries) {
|
|
263
|
+
const n = asEntry(e)
|
|
264
|
+
if (n) store.entries.push(n)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
if (Array.isArray(parsed.captures)) store.captures = parsed.captures.slice(-CAPTURE_MAX)
|
|
268
|
+
if (Array.isArray(parsed.log)) store.log = parsed.log.slice(-LOG_MAX)
|
|
269
|
+
if (typeof parsed.logSeq === 'number') store.logSeq = parsed.logSeq
|
|
270
|
+
if (parsed.meta && typeof parsed.meta === 'object') {
|
|
271
|
+
store.meta.collectionReady = parsed.meta.collectionReady === true
|
|
272
|
+
store.meta.dimension = intOf(parsed.meta.dimension, 0)
|
|
273
|
+
store.meta.seeded = parsed.meta.seeded === true
|
|
274
|
+
store.meta.indexed = intOf(parsed.meta.indexed, 0)
|
|
275
|
+
}
|
|
276
|
+
recalcMeta()
|
|
277
|
+
}
|
|
278
|
+
store.meta.persistence = 'ready'
|
|
279
|
+
return store.entries.length
|
|
280
|
+
} catch (e) {
|
|
281
|
+
const m = msgOf(e)
|
|
282
|
+
if (!/ENOENT|not found|不存在|null/i.test(m)) {
|
|
283
|
+
store.meta.persistence = 'error'
|
|
284
|
+
store.meta.lastError = '读取记忆库失败:' + m
|
|
285
|
+
log('err', store.meta.lastError)
|
|
286
|
+
} else {
|
|
287
|
+
store.meta.persistence = 'ready'
|
|
288
|
+
}
|
|
289
|
+
return 0
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function hostPathOf(fs, target) {
|
|
294
|
+
try {
|
|
295
|
+
if (typeof fs.processPath === 'function') return String(fs.processPath(target) || '')
|
|
296
|
+
} catch (e) { /* 沙箱实现没有 processPath 时留空 */ }
|
|
297
|
+
return ''
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function persist() {
|
|
301
|
+
store.updatedAt = nowMs()
|
|
302
|
+
const r = await resolveTarget()
|
|
303
|
+
if (!r) return false
|
|
304
|
+
try {
|
|
305
|
+
const payload = {
|
|
306
|
+
version: STORE_VERSION,
|
|
307
|
+
updatedAt: store.updatedAt,
|
|
308
|
+
settings: store.settings,
|
|
309
|
+
entries: store.entries.slice(-ENTRY_MAX),
|
|
310
|
+
captures: store.captures.slice(-CAPTURE_MAX),
|
|
311
|
+
log: store.log.slice(-LOG_MAX),
|
|
312
|
+
logSeq: store.logSeq,
|
|
313
|
+
meta: {
|
|
314
|
+
collectionReady: store.meta.collectionReady,
|
|
315
|
+
dimension: store.meta.dimension,
|
|
316
|
+
seeded: store.meta.seeded,
|
|
317
|
+
indexed: store.meta.indexed,
|
|
318
|
+
},
|
|
319
|
+
}
|
|
320
|
+
await r.fs.writeText(r.target, JSON.stringify(payload, null, 2))
|
|
321
|
+
store.meta.storePath = hostPathOf(r.fs, r.target)
|
|
322
|
+
store.meta.persistence = 'ready'
|
|
323
|
+
return true
|
|
324
|
+
} catch (e) {
|
|
325
|
+
store.meta.persistence = 'error'
|
|
326
|
+
store.meta.lastError = '写入失败:' + msgOf(e)
|
|
327
|
+
log('err', store.meta.lastError)
|
|
328
|
+
return false
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
async function ensureLoaded() { if (loaded) return 0; loaded = true; return await doLoad() }
|
|
333
|
+
|
|
334
|
+
// ── HTTP:统一走 shell + curl ─────────────────────────────────────────────
|
|
335
|
+
async function curlJson(url, opts) {
|
|
336
|
+
const o = opts || {}
|
|
337
|
+
const shell = ctx.get('shell')
|
|
338
|
+
if (shell === undefined || shell === null) throw new Error('shell 服务不可用(HTTP 走 curl,需要 shell 服务)')
|
|
339
|
+
const t = intOf(o.timeoutMs, HTTP_TIMEOUT_MS)
|
|
340
|
+
const parts = ['curl -sS -m ' + Math.ceil(t / 1000), '-X ' + (o.method || 'POST')]
|
|
341
|
+
for (const h of (o.headers || [])) parts.push('-H ' + shQuote(h))
|
|
342
|
+
if (o.body !== undefined && o.body !== null) parts.push('--data-binary ' + shQuote(typeof o.body === 'string' ? o.body : JSON.stringify(o.body)))
|
|
343
|
+
parts.push(shQuote(url))
|
|
344
|
+
// 追加 HTTP 状态码:curl 成功但 HTTP 失败时才能给出准确原因(Milvus 的业务错误反而是 200)
|
|
345
|
+
parts.push('-w ' + shQuote('\\n%{http_code}'))
|
|
346
|
+
const spec = shell.resolve({ command: parts.join(' '), timeoutMs: t + 8000, stdoutMaxBytes: intOf(o.maxBytes, 4000000) })
|
|
347
|
+
const res = await shell.run(spec)
|
|
348
|
+
const raw = res && res.stdout ? String(res.stdout.text || '') : ''
|
|
349
|
+
const errText = res && res.stderr ? String(res.stderr.text || '') : ''
|
|
350
|
+
if (res && res.timedOut) throw new Error('curl 超时(' + Math.round(t / 1000) + 's)')
|
|
351
|
+
const cut = raw.lastIndexOf('\n')
|
|
352
|
+
const body = cut >= 0 ? raw.slice(0, cut) : raw
|
|
353
|
+
const code = cut >= 0 ? Number(raw.slice(cut + 1).trim()) : 0
|
|
354
|
+
if (res && res.exitCode !== 0) throw new Error('curl 退出码 ' + res.exitCode + ':' + clip(errText || body, 240))
|
|
355
|
+
if (!(code >= 200 && code < 300)) throw new Error('HTTP ' + code + ':' + clip(body || errText, 400))
|
|
356
|
+
if (!String(body).trim()) return null
|
|
357
|
+
try { return JSON.parse(body) } catch (e) { throw new Error('响应不是 JSON:' + clip(body, 300)) }
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// ── 向量模型 ──────────────────────────────────────────────────────────────
|
|
361
|
+
function embedPreset(s) { return EMBED_PRESETS[s.embed.provider] || EMBED_PRESETS.dashscope }
|
|
362
|
+
function embedBase(s) { return String(s.embed.baseUrl || '').trim().replace(/\/+$/, '') || embedPreset(s).base }
|
|
363
|
+
function embedHeaders(s) {
|
|
364
|
+
const h = ['Content-Type: application/json']
|
|
365
|
+
if (String(s.embed.apiKey || '').trim()) h.push('Authorization: Bearer ' + String(s.embed.apiKey).trim())
|
|
366
|
+
return h
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async function embedBatch(texts, s) {
|
|
370
|
+
const cfg = s || settings()
|
|
371
|
+
const provider = String(cfg.embed.provider || 'dashscope')
|
|
372
|
+
const model = String(cfg.embed.model || '').trim()
|
|
373
|
+
if (!model) throw new Error('未配置向量模型')
|
|
374
|
+
const base = embedBase(cfg)
|
|
375
|
+
if (!base) throw new Error('未配置向量模型的接口地址(baseUrl)')
|
|
376
|
+
if (!String(cfg.embed.apiKey || '').trim() && provider !== 'openai') {
|
|
377
|
+
log('warn', '向量模型未填 API Key(' + provider + '):如果服务需要鉴权会返回 401')
|
|
378
|
+
}
|
|
379
|
+
let url, body, pick
|
|
380
|
+
|
|
381
|
+
if (provider === 'dashscope') {
|
|
382
|
+
// 多模态与纯文本是两个端点,按模型名判断
|
|
383
|
+
const isMultimodal = /vision|multimodal/i.test(model)
|
|
384
|
+
url = base + (isMultimodal
|
|
385
|
+
? '/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding'
|
|
386
|
+
: '/api/v1/services/embeddings/text-embedding/text-embedding')
|
|
387
|
+
body = isMultimodal
|
|
388
|
+
? { model: model, input: { contents: texts.map(function (x) { return { text: x } }) }, parameters: { dimension: cfg.embed.dimension } }
|
|
389
|
+
: { model: model, input: { texts: texts }, parameters: { dimension: cfg.embed.dimension, text_type: 'document' } }
|
|
390
|
+
pick = function (res) {
|
|
391
|
+
const out = res && res.output && res.output.embeddings
|
|
392
|
+
if (!Array.isArray(out)) throw new Error('DashScope 响应里没有 output.embeddings:' + clip(JSON.stringify(res), 300))
|
|
393
|
+
return out.map(function (e) { return e.embedding })
|
|
394
|
+
}
|
|
395
|
+
} else if (provider === 'bigmodel') {
|
|
396
|
+
url = base + '/api/paas/v4/embeddings'
|
|
397
|
+
body = { model: model, input: texts }
|
|
398
|
+
if (cfg.embed.dimension) body.dimensions = cfg.embed.dimension
|
|
399
|
+
pick = function (res) {
|
|
400
|
+
const out = res && res.data
|
|
401
|
+
if (!Array.isArray(out)) throw new Error('BigModel 响应里没有 data:' + clip(JSON.stringify(res), 300))
|
|
402
|
+
return out.slice().sort(function (a, b) { return intOf(a.index, 0) - intOf(b.index, 0) }).map(function (e) { return e.embedding })
|
|
403
|
+
}
|
|
404
|
+
} else {
|
|
405
|
+
url = base + '/v1/embeddings'
|
|
406
|
+
body = { model: model, input: texts }
|
|
407
|
+
if (cfg.embed.dimension && /text-embedding-3/.test(model)) body.dimensions = cfg.embed.dimension
|
|
408
|
+
pick = function (res) {
|
|
409
|
+
const out = res && res.data
|
|
410
|
+
if (!Array.isArray(out)) throw new Error('OpenAI 兼容响应里没有 data:' + clip(JSON.stringify(res), 300))
|
|
411
|
+
return out.slice().sort(function (a, b) { return intOf(a.index, 0) - intOf(b.index, 0) }).map(function (e) { return e.embedding })
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
const res = await curlJson(url, { body: body, headers: embedHeaders(cfg), timeoutMs: HTTP_TIMEOUT_MS })
|
|
416
|
+
const vecs = pick(res)
|
|
417
|
+
for (const v of vecs) if (!Array.isArray(v) || v.length === 0) throw new Error('返回了空向量(模型 ' + model + ')')
|
|
418
|
+
if (vecs.length !== texts.length) throw new Error('返回向量数(' + vecs.length + ')与输入条数(' + texts.length + ')不一致')
|
|
419
|
+
return vecs
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
async function embedTexts(texts, s) {
|
|
423
|
+
const out = []
|
|
424
|
+
for (let i = 0; i < texts.length; i += EMBED_BATCH) {
|
|
425
|
+
const chunk = texts.slice(i, i + EMBED_BATCH)
|
|
426
|
+
const vecs = await embedBatch(chunk, s)
|
|
427
|
+
for (const v of vecs) out.push(v)
|
|
428
|
+
store.meta.progress = { text: '向量化 ' + Math.min(i + chunk.length, texts.length) + '/' + texts.length, done: i + chunk.length, total: texts.length }
|
|
429
|
+
}
|
|
430
|
+
return out
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// ── 重排模型 ──────────────────────────────────────────────────────────────
|
|
434
|
+
function rerankPreset(s) { return RERANK_PRESETS[s.rerank.provider] || RERANK_PRESETS.jina }
|
|
435
|
+
function rerankReady(s) {
|
|
436
|
+
const cfg = s || settings()
|
|
437
|
+
return cfg.rerank.enabled === true && !!String(cfg.rerank.model || '').trim() && !!String(cfg.rerank.provider || '').trim()
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function rerankDocs(query, docs, s) {
|
|
441
|
+
const cfg = s || settings()
|
|
442
|
+
const preset = rerankPreset(cfg)
|
|
443
|
+
const base = String(cfg.rerank.baseUrl || '').trim().replace(/\/+$/, '') || preset.base
|
|
444
|
+
if (!base) throw new Error('未配置重排服务的接口地址(baseUrl)')
|
|
445
|
+
const url = base + (preset.path || '/v1/rerank')
|
|
446
|
+
const headers = ['Content-Type: application/json']
|
|
447
|
+
if (String(cfg.rerank.apiKey || '').trim()) headers.push('Authorization: Bearer ' + String(cfg.rerank.apiKey).trim())
|
|
448
|
+
const topN = Math.max(1, Math.min(docs.length, intOf(cfg.rerank.topN, 8)))
|
|
449
|
+
const body = cfg.rerank.provider === 'dashscope'
|
|
450
|
+
? { model: cfg.rerank.model, input: { query: query, documents: docs }, parameters: { return_documents: false, top_n: topN } }
|
|
451
|
+
: { model: cfg.rerank.model, query: query, documents: docs, top_n: topN }
|
|
452
|
+
const res = await curlJson(url, { body: body, headers: headers, timeoutMs: HTTP_TIMEOUT_MS })
|
|
453
|
+
// 字段位置各家不同:多数在 results,百炼在 output.results;分数多为 relevance_score
|
|
454
|
+
const list = (res && Array.isArray(res.results)) ? res.results
|
|
455
|
+
: ((res && res.output && Array.isArray(res.output.results)) ? res.output.results : null)
|
|
456
|
+
if (!list) throw new Error('重排响应里没有 results:' + clip(JSON.stringify(res), 300))
|
|
457
|
+
const out = []
|
|
458
|
+
for (const r of list) {
|
|
459
|
+
const idx = intOf(r.index, -1)
|
|
460
|
+
if (idx < 0 || idx >= docs.length) continue
|
|
461
|
+
out.push({ index: idx, score: Number(r.relevance_score !== undefined ? r.relevance_score : r.score) || 0 })
|
|
462
|
+
}
|
|
463
|
+
return out
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// ── Milvus(REST v2) ─────────────────────────────────────────────────────
|
|
467
|
+
function milvusCfg(s) { return (s || settings()).milvus }
|
|
468
|
+
function milvusBase(s) { return String(milvusCfg(s).uri || '').trim().replace(/\/+$/, '') }
|
|
469
|
+
function collectionName(s) { return String(milvusCfg(s).collection || 'redteam_memory').trim() || 'redteam_memory' }
|
|
470
|
+
|
|
471
|
+
function milvusUrl(path, s) {
|
|
472
|
+
const cfg = milvusCfg(s)
|
|
473
|
+
const base = milvusBase(s)
|
|
474
|
+
if (!base) throw new Error('未配置 Milvus 地址(自建如 http://127.0.0.1:19530,Zilliz 如 https://xxx.api.region.zillizcloud.com)')
|
|
475
|
+
let url = base + path
|
|
476
|
+
if (cfg.dbName) url += (path.indexOf('?') >= 0 ? '&' : '?') + 'dbName=' + encodeURIComponent(String(cfg.dbName))
|
|
477
|
+
return url
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
async function milvusCall(path, body, s) {
|
|
481
|
+
const cfg = milvusCfg(s)
|
|
482
|
+
const headers = ['Content-Type: application/json', 'Accept: application/json']
|
|
483
|
+
if (String(cfg.token || '').trim()) headers.push('Authorization: Bearer ' + String(cfg.token).trim())
|
|
484
|
+
const res = await curlJson(milvusUrl('/v2/vectordb' + path, s), { body: body || {}, headers: headers, timeoutMs: HTTP_TIMEOUT_MS })
|
|
485
|
+
// Milvus v2 的业务错误是 HTTP 200 + code != 0
|
|
486
|
+
if (res && typeof res.code === 'number' && res.code !== 0) {
|
|
487
|
+
throw new Error('Milvus 错误 code=' + res.code + ':' + clip(res.message || JSON.stringify(res), 300))
|
|
488
|
+
}
|
|
489
|
+
return res
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
async function milvusListCollections(s) {
|
|
493
|
+
const res = await milvusCall('/collections/list', {}, s)
|
|
494
|
+
const d = res && res.data
|
|
495
|
+
if (Array.isArray(d)) return d.map(String)
|
|
496
|
+
if (d && Array.isArray(d.collectionNames)) return d.collectionNames.map(String)
|
|
497
|
+
return []
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async function milvusDescribe(s) {
|
|
501
|
+
const res = await milvusCall('/collections/describe', { collectionName: collectionName(s) }, s)
|
|
502
|
+
return res && res.data ? res.data : null
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// 显式 schema(不用动态字段):字段类型可控,查询结果稳定。
|
|
506
|
+
function buildSchema(dim) {
|
|
507
|
+
return {
|
|
508
|
+
autoId: false,
|
|
509
|
+
enabledDynamicField: false,
|
|
510
|
+
fields: [
|
|
511
|
+
{ fieldName: 'id', dataType: 'VarChar', isPrimary: true, maxLength: 64 },
|
|
512
|
+
{ fieldName: 'vector', dataType: 'FloatVector', dim: dim },
|
|
513
|
+
{ fieldName: 'title', dataType: 'VarChar', maxLength: 512 },
|
|
514
|
+
{ fieldName: 'text', dataType: 'VarChar', maxLength: 16384 },
|
|
515
|
+
{ fieldName: 'tags', dataType: 'VarChar', maxLength: 512 },
|
|
516
|
+
{ fieldName: 'kind', dataType: 'VarChar', maxLength: 32 },
|
|
517
|
+
{ fieldName: 'source', dataType: 'VarChar', maxLength: 512 },
|
|
518
|
+
{ fieldName: 'created_at', dataType: 'Int64' },
|
|
519
|
+
{ fieldName: 'updated_at', dataType: 'Int64' },
|
|
520
|
+
],
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function dimOfDescribe(d) {
|
|
525
|
+
const fields = (d && d.fields) || []
|
|
526
|
+
for (const f of fields) {
|
|
527
|
+
const name = String(f.name || f.fieldName || '')
|
|
528
|
+
if (name !== 'vector') continue
|
|
529
|
+
if (f.params && f.params.dim !== undefined) return intOf(f.params.dim, 0)
|
|
530
|
+
if (f.dim !== undefined) return intOf(f.dim, 0)
|
|
531
|
+
}
|
|
532
|
+
return 0
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
async function milvusEnsureCollection(s, dim) {
|
|
536
|
+
const name = collectionName(s)
|
|
537
|
+
const list = await milvusListCollections(s)
|
|
538
|
+
if (list.indexOf(name) >= 0) {
|
|
539
|
+
const existing = dimOfDescribe(await milvusDescribe(s))
|
|
540
|
+
if (existing && dim && existing !== dim) {
|
|
541
|
+
throw new Error('集合 ' + name + ' 已存在,向量维度是 ' + existing + ',与当前模型维度 ' + dim + ' 不一致。换模型必须换集合名,或先删掉集合重建。')
|
|
542
|
+
}
|
|
543
|
+
store.meta.collectionReady = true
|
|
544
|
+
store.meta.dimension = existing || dim || 0
|
|
545
|
+
return { created: false, name: name, dimension: store.meta.dimension }
|
|
546
|
+
}
|
|
547
|
+
await milvusCall('/collections/create', {
|
|
548
|
+
collectionName: name,
|
|
549
|
+
schema: buildSchema(dim),
|
|
550
|
+
indexParams: [{ fieldName: 'vector', indexName: 'vector_idx', metricType: String(milvusCfg(s).metric || 'COSINE'), indexType: 'AUTOINDEX' }],
|
|
551
|
+
}, s)
|
|
552
|
+
store.meta.collectionReady = true
|
|
553
|
+
store.meta.dimension = dim
|
|
554
|
+
return { created: true, name: name, dimension: dim }
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
async function milvusSearch(vector, topK, s, filter) {
|
|
558
|
+
const body = {
|
|
559
|
+
collectionName: collectionName(s),
|
|
560
|
+
data: [vector],
|
|
561
|
+
limit: Math.max(1, Math.min(200, intOf(topK, 8))),
|
|
562
|
+
outputFields: ['id', 'title', 'text', 'tags', 'kind', 'source', 'created_at'],
|
|
563
|
+
}
|
|
564
|
+
if (filter) body.filter = filter
|
|
565
|
+
const res = await milvusCall('/entities/search', body, s)
|
|
566
|
+
const d = res && res.data
|
|
567
|
+
if (Array.isArray(d)) return d
|
|
568
|
+
if (d && Array.isArray(d.data)) return d.data
|
|
569
|
+
return []
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
async function milvusDelete(ids, s) {
|
|
573
|
+
if (!ids.length) return 0
|
|
574
|
+
const quoted = ids.map(function (x) { return '"' + String(x).replace(/"/g, '') + '"' })
|
|
575
|
+
const res = await milvusCall('/entities/delete', { collectionName: collectionName(s), filter: 'id in [' + quoted.join(',') + ']' }, s)
|
|
576
|
+
const d = res && res.data
|
|
577
|
+
return intOf(d && (d.deleteCount !== undefined ? d.deleteCount : d.delete_count), ids.length)
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
async function milvusDrop(s) {
|
|
581
|
+
await milvusCall('/collections/drop', { collectionName: collectionName(s) }, s)
|
|
582
|
+
store.meta.collectionReady = false
|
|
583
|
+
store.meta.rowCount = 0
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
async function milvusCount(s) {
|
|
587
|
+
try {
|
|
588
|
+
const res = await milvusCall('/collections/get_stats', { collectionName: collectionName(s) }, s)
|
|
589
|
+
const d = res && res.data
|
|
590
|
+
const n = intOf(d && (d.rowCount !== undefined ? d.rowCount : d.row_count), -1)
|
|
591
|
+
if (n >= 0) return n
|
|
592
|
+
} catch (e) { /* 旧版本可能没有 get_stats,退回 query 统计 */ }
|
|
593
|
+
try {
|
|
594
|
+
const res = await milvusCall('/entities/query', { collectionName: collectionName(s), filter: '', outputFields: ['count(*)'], limit: 1 }, s)
|
|
595
|
+
const d = res && res.data
|
|
596
|
+
const first = Array.isArray(d) ? d[0] : (d && Array.isArray(d.data) ? d.data[0] : null)
|
|
597
|
+
if (first) return intOf(first['count(*)'] !== undefined ? first['count(*)'] : first.count, 0)
|
|
598
|
+
} catch (e) { /* 统计拿不到不影响主流程 */ }
|
|
599
|
+
return -1
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// ── MinIO(Milvus 的存储桶,例如 mimo):curl 自带 --aws-sigv4 ────────────────
|
|
603
|
+
function s3Cfg(s) { return (s || settings()).s3 }
|
|
604
|
+
function s3Ready(s) {
|
|
605
|
+
const c = s3Cfg(s)
|
|
606
|
+
return c.enabled === true && !!String(c.endpoint || '').trim() && !!String(c.bucket || '').trim()
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
function s3Url(key, query, s) {
|
|
610
|
+
const c = s3Cfg(s)
|
|
611
|
+
const base = String(c.endpoint).replace(/\/+$/, '')
|
|
612
|
+
const path = c.pathStyle === false
|
|
613
|
+
? base.replace('://', '://' + c.bucket + '.') + (key ? '/' + key : '/')
|
|
614
|
+
: base + '/' + c.bucket + (key ? '/' + key : '')
|
|
615
|
+
return query ? path + '?' + query : path
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async function s3Call(method, key, query, body, s, opts) {
|
|
619
|
+
const c = s3Cfg(s)
|
|
620
|
+
if (!String(c.endpoint || '').trim()) throw new Error('未配置 MinIO 端点')
|
|
621
|
+
const shell = ctx.get('shell')
|
|
622
|
+
if (shell === undefined || shell === null) throw new Error('shell 服务不可用(MinIO 走 curl)')
|
|
623
|
+
const region = String(c.region || 'us-east-1')
|
|
624
|
+
const t = intOf(opts && opts.timeoutMs, HTTP_TIMEOUT_MS)
|
|
625
|
+
const parts = ['curl -sS -m ' + Math.ceil(t / 1000), '-X ' + method]
|
|
626
|
+
parts.push('--aws-sigv4 ' + shQuote('aws:amz:' + region + ':s3'))
|
|
627
|
+
parts.push('--user ' + shQuote(String(c.accessKey || '') + ':' + String(c.secretKey || '')))
|
|
628
|
+
if (body !== undefined && body !== null) {
|
|
629
|
+
parts.push('-H ' + shQuote('Content-Type: ' + ((opts && opts.contentType) || 'application/octet-stream')))
|
|
630
|
+
parts.push('--data-binary ' + shQuote(body))
|
|
631
|
+
}
|
|
632
|
+
parts.push(shQuote(s3Url(key, query, s)))
|
|
633
|
+
parts.push('-w ' + shQuote('\\n%{http_code}'))
|
|
634
|
+
const spec = shell.resolve({ command: parts.join(' '), timeoutMs: t + 8000, stdoutMaxBytes: intOf(opts && opts.maxBytes, 4000000) })
|
|
635
|
+
const res = await shell.run(spec)
|
|
636
|
+
const raw = res && res.stdout ? String(res.stdout.text || '') : ''
|
|
637
|
+
const errText = res && res.stderr ? String(res.stderr.text || '') : ''
|
|
638
|
+
if (res && res.timedOut) throw new Error('MinIO 请求超时')
|
|
639
|
+
const cut = raw.lastIndexOf('\n')
|
|
640
|
+
const text = cut >= 0 ? raw.slice(0, cut) : raw
|
|
641
|
+
const code = cut >= 0 ? Number(raw.slice(cut + 1).trim()) : 0
|
|
642
|
+
if (res && res.exitCode !== 0) throw new Error('curl 退出码 ' + res.exitCode + ':' + clip(errText || text, 240))
|
|
643
|
+
if (!(code >= 200 && code < 300)) throw new Error('MinIO HTTP ' + code + ':' + clip(text || errText, 400))
|
|
644
|
+
return text
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// ListObjectsV2 返回 XML:只需要 Key/Size/LastModified,正则足够,不必引 XML 解析器。
|
|
648
|
+
function parseS3List(xml) {
|
|
649
|
+
const out = []
|
|
650
|
+
const re = /<Contents>([\s\S]*?)<\/Contents>/g
|
|
651
|
+
let m
|
|
652
|
+
while ((m = re.exec(xml))) {
|
|
653
|
+
const block = m[1]
|
|
654
|
+
const pick = function (tag) {
|
|
655
|
+
const r = new RegExp('<' + tag + '>([\\s\\S]*?)</' + tag + '>')
|
|
656
|
+
const mm = r.exec(block)
|
|
657
|
+
return mm ? mm[1].replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') : ''
|
|
658
|
+
}
|
|
659
|
+
out.push({ key: pick('Key'), size: intOf(pick('Size'), 0), lastModified: pick('LastModified'), storageClass: pick('StorageClass') })
|
|
660
|
+
}
|
|
661
|
+
const tokM = /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/.exec(xml)
|
|
662
|
+
return { objects: out, truncated: /<IsTruncated>true<\/IsTruncated>/.test(xml), nextToken: tokM ? tokM[1] : '' }
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
async function s3List(prefix, limit, s) {
|
|
666
|
+
const q = ['list-type=2', 'max-keys=' + Math.max(1, Math.min(1000, intOf(limit, 50)))]
|
|
667
|
+
const p = String(prefix || '').trim()
|
|
668
|
+
if (p) q.push('prefix=' + encodeURIComponent(p))
|
|
669
|
+
return parseS3List(await s3Call('GET', '', q.join('&'), null, s))
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// ── 知识库操作 ────────────────────────────────────────────────────────────
|
|
673
|
+
// ── 导入解析:pdf / word(.docx) / md / txt ─────────────────────────────────
|
|
674
|
+
function extOf(path) {
|
|
675
|
+
const m = /\.([A-Za-z0-9]+)\s*$/.exec(String(path || '').trim())
|
|
676
|
+
return m ? '.' + m[1].toLowerCase() : ''
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// Word 的正文就是一个 XML:段落边界是 </w:p>,制表与换行有专门标签。
|
|
680
|
+
// 抽文本只需要这三条规则 + 反转义,不需要完整 XML 解析器。
|
|
681
|
+
function docxXmlToText(xml) {
|
|
682
|
+
let s = String(xml || '')
|
|
683
|
+
s = s.replace(/<w:tab\b[^>]*\/?>/g, '\t')
|
|
684
|
+
s = s.replace(/<w:br\b[^>]*\/?>/g, '\n')
|
|
685
|
+
s = s.replace(/<\/w:p>/g, '\n')
|
|
686
|
+
s = s.replace(/<[^>]+>/g, '')
|
|
687
|
+
s = s.replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&')
|
|
688
|
+
return s.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').trim()
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
function splitPlainText(text) {
|
|
692
|
+
const out = []
|
|
693
|
+
let buf = []
|
|
694
|
+
let size = 0
|
|
695
|
+
const flush = function () {
|
|
696
|
+
if (!buf.length) return
|
|
697
|
+
const t = buf.join('\n').trim()
|
|
698
|
+
if (t) out.push(t)
|
|
699
|
+
buf = []
|
|
700
|
+
size = 0
|
|
701
|
+
}
|
|
702
|
+
for (const para of String(text || '').split(/\n\s*\n/)) {
|
|
703
|
+
const p = para.trim()
|
|
704
|
+
if (!p) continue
|
|
705
|
+
// 已经攒了内容、再加就超上限时先收一段:这样段落边界才是切点,
|
|
706
|
+
// 而不是「撞到上限才切」(那会把一整段正文和前面的短标题糊在一起)。
|
|
707
|
+
if (size > 0 && size + p.length > PLAIN_GROUP_MAX) flush()
|
|
708
|
+
buf.push(p)
|
|
709
|
+
size += p.length
|
|
710
|
+
if (size >= PLAIN_GROUP_MAX) flush()
|
|
711
|
+
}
|
|
712
|
+
flush()
|
|
713
|
+
return out
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
async function readImport(path) {
|
|
717
|
+
const ext = extOf(path)
|
|
718
|
+
if (IMPORT_EXT.indexOf(ext) < 0) {
|
|
719
|
+
throw new Error('只支持 ' + IMPORT_LABEL + '(拿到的是 ' + (ext ? ext + ' 文件' : '没有扩展名的文件') + ')')
|
|
720
|
+
}
|
|
721
|
+
if (ext === '.md' || ext === '.markdown' || ext === '.txt') {
|
|
722
|
+
const fs = ctx.get('fs')
|
|
723
|
+
if (fs === undefined || fs === null) throw new Error('fs 服务不可用,读不了文本文件')
|
|
724
|
+
const target = await fs.resolve(path)
|
|
725
|
+
const info = await fs.stat(target)
|
|
726
|
+
if (!info) throw new Error('文件不存在:' + path)
|
|
727
|
+
return { text: String(await fs.readText(target)), ext: ext, how: '直接读文本' }
|
|
728
|
+
}
|
|
729
|
+
const shell = ctx.get('shell')
|
|
730
|
+
if (shell === undefined || shell === null || typeof shell.resolve !== 'function') {
|
|
731
|
+
throw new Error('shell 服务不可用:' + (ext === '.docx' ? 'word' : 'pdf') + ' 要靠外部命令抽文本')
|
|
732
|
+
}
|
|
733
|
+
const cmd = ext === '.docx'
|
|
734
|
+
? 'unzip -p ' + shQuote(path) + ' word/document.xml'
|
|
735
|
+
: 'pdftotext -layout ' + shQuote(path) + ' -'
|
|
736
|
+
const spec = shell.resolve({ command: cmd, timeoutMs: 60000, stdoutMaxBytes: 16 * 1024 * 1024 })
|
|
737
|
+
const r = await shell.run(spec)
|
|
738
|
+
const body = (r && r.stdout && r.stdout.text) || ''
|
|
739
|
+
if (!r || r.exitCode !== 0 || !String(body).trim()) {
|
|
740
|
+
const why = clip((r && r.stderr && r.stderr.text) || ('命令退出码 ' + (r && r.exitCode)), 200)
|
|
741
|
+
throw new Error((ext === '.docx' ? 'word' : 'pdf') + ' 抽文本失败:' + why
|
|
742
|
+
+ (ext === '.pdf' ? '(pdf 需要 poppler 的 pdftotext)' : '(老式 .doc 不支持,请先另存为 .docx)'))
|
|
743
|
+
}
|
|
744
|
+
return { text: ext === '.docx' ? docxXmlToText(body) : String(body), ext: ext, how: ext === '.docx' ? 'unzip 取 word/document.xml' : 'pdftotext -layout' }
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
// 切条目:markdown 按 ## 分节;word / pdf 按长度归拢。
|
|
748
|
+
function entriesFromDoc(path, doc) {
|
|
749
|
+
const tag = doc.ext.replace('.', '')
|
|
750
|
+
const out = []
|
|
751
|
+
if (doc.ext === '.md' || doc.ext === '.markdown') {
|
|
752
|
+
for (const part of String(doc.text).split(/\n(?=##\s)/)) {
|
|
753
|
+
const t = part.trim()
|
|
754
|
+
if (!t) continue
|
|
755
|
+
const m = /^##\s+(.+)$/m.exec(t)
|
|
756
|
+
out.push({ title: clip((m ? m[1] : t.split('\n')[0]) || path, 120), text: t, kind: 'knowledge', tags: [tag], source: path })
|
|
757
|
+
}
|
|
758
|
+
} else {
|
|
759
|
+
const groups = splitPlainText(doc.text)
|
|
760
|
+
for (let i = 0; i < groups.length; i++) {
|
|
761
|
+
const t = groups[i]
|
|
762
|
+
out.push({ title: clip(t.split('\n')[0] || (path + ' 第 ' + (i + 1) + ' 段'), 120), text: t, kind: 'knowledge', tags: [tag], source: path })
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
return out
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
function splitChunks(text) {
|
|
769
|
+
const s = String(text || '').trim()
|
|
770
|
+
if (!s) return []
|
|
771
|
+
if (s.length <= CHUNK_MAX) return [s]
|
|
772
|
+
const out = []
|
|
773
|
+
let i = 0
|
|
774
|
+
while (i < s.length) {
|
|
775
|
+
out.push(s.slice(i, i + CHUNK_MAX))
|
|
776
|
+
if (i + CHUNK_MAX >= s.length) break
|
|
777
|
+
i += CHUNK_MAX - CHUNK_OVERLAP
|
|
778
|
+
}
|
|
779
|
+
return out
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
function asEntry(raw, fallbackSource) {
|
|
783
|
+
const r = raw && typeof raw === 'object' ? raw : {}
|
|
784
|
+
const text = String(r.text === undefined ? '' : r.text).trim()
|
|
785
|
+
if (!text) return null
|
|
786
|
+
const e = {
|
|
787
|
+
title: clip(r.title || text.split('\n')[0], 200),
|
|
788
|
+
text: text,
|
|
789
|
+
kind: KINDS.indexOf(String(r.kind)) >= 0 ? String(r.kind) : 'knowledge',
|
|
790
|
+
tags: Array.isArray(r.tags)
|
|
791
|
+
? r.tags.map(String).slice(0, 12)
|
|
792
|
+
: String(r.tags || '').split(/[,,\s]+/).filter(Boolean).slice(0, 12),
|
|
793
|
+
source: clip(r.source || fallbackSource || 'manual', 200),
|
|
794
|
+
created_at: intOf(r.created_at, 0) || nowMs(),
|
|
795
|
+
updated_at: intOf(r.updated_at, 0) || nowMs(),
|
|
796
|
+
indexed: r.indexed === true,
|
|
797
|
+
}
|
|
798
|
+
e.id = String(r.id || '') || contentId(['e', e.kind, e.title, e.text])
|
|
799
|
+
return e
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
function indexOfEntry(id) {
|
|
803
|
+
for (let i = 0; i < store.entries.length; i++) if (store.entries[i].id === id) return i
|
|
804
|
+
return -1
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function recalcMeta() {
|
|
808
|
+
let indexed = 0
|
|
809
|
+
for (const e of store.entries) if (e.indexed === true) indexed++
|
|
810
|
+
store.meta.localCount = store.entries.length
|
|
811
|
+
store.meta.indexed = indexed
|
|
812
|
+
store.meta.pending = store.entries.length - indexed
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// 条目 -> 索引行。行 id 以条目 id 开头(`<条目 id>#<块号>-<内容指纹>`),
|
|
816
|
+
// 于是从 Milvus 的检索结果能直接反查回本地条目,删除也不必回查向量库。
|
|
817
|
+
function chunkRowsOf(e) {
|
|
818
|
+
const chunks = splitChunks(e.text)
|
|
819
|
+
const rows = []
|
|
820
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
821
|
+
const part = chunks[i]
|
|
822
|
+
rows.push({
|
|
823
|
+
id: e.id + '#' + i + '-' + contentId([e.id, String(i), part]).slice(1, 9),
|
|
824
|
+
title: chunks.length > 1 ? e.title + '(' + (i + 1) + '/' + chunks.length + ')' : e.title,
|
|
825
|
+
text: part,
|
|
826
|
+
tags: (e.tags || []).join(','),
|
|
827
|
+
kind: e.kind,
|
|
828
|
+
source: e.source,
|
|
829
|
+
created_at: e.created_at,
|
|
830
|
+
updated_at: e.updated_at,
|
|
831
|
+
})
|
|
832
|
+
}
|
|
833
|
+
return rows
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
function entryIdFromRow(id) { return String(id || '').split('#')[0] }
|
|
837
|
+
|
|
838
|
+
// 本地写入:同 id(同标题同正文)视为同一段知识,覆盖而不是重复累积。
|
|
839
|
+
function putEntriesLocal(entries) {
|
|
840
|
+
let added = 0, updated = 0
|
|
841
|
+
for (const e of entries) {
|
|
842
|
+
const at = indexOfEntry(e.id)
|
|
843
|
+
if (at >= 0) {
|
|
844
|
+
e.created_at = store.entries[at].created_at || e.created_at
|
|
845
|
+
e.indexed = false
|
|
846
|
+
store.entries[at] = e
|
|
847
|
+
updated++
|
|
848
|
+
} else {
|
|
849
|
+
store.entries.push(e)
|
|
850
|
+
added++
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
if (store.entries.length > ENTRY_MAX) {
|
|
854
|
+
const drop = store.entries.length - ENTRY_MAX
|
|
855
|
+
store.entries = store.entries.slice(drop)
|
|
856
|
+
log('warn', '本地条目超过上限 ' + ENTRY_MAX + ',丢弃最旧的 ' + drop + ' 条')
|
|
857
|
+
}
|
|
858
|
+
recalcMeta()
|
|
859
|
+
return { added: added, updated: updated }
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
// 把条目同步进 Milvus。可能抛错(没 Key / 连不上),由调用方决定怎么记。
|
|
863
|
+
async function indexEntries(list, s) {
|
|
864
|
+
const cfg = s || settings()
|
|
865
|
+
const rows = []
|
|
866
|
+
for (const e of list) for (const r of chunkRowsOf(e)) rows.push(r)
|
|
867
|
+
if (!rows.length) return { rows: 0, dimension: 0 }
|
|
868
|
+
const vectors = await embedTexts(rows.map(function (r) { return r.title + '\n' + r.text }), cfg)
|
|
869
|
+
for (let i = 0; i < rows.length; i++) rows[i].vector = vectors[i]
|
|
870
|
+
const dim = vectors[0].length
|
|
871
|
+
await milvusEnsureCollection(cfg, dim)
|
|
872
|
+
let n = 0
|
|
873
|
+
for (let i = 0; i < rows.length; i += INSERT_BATCH) {
|
|
874
|
+
const batch = rows.slice(i, i + INSERT_BATCH)
|
|
875
|
+
try {
|
|
876
|
+
// upsert:同 id 重复导入时覆盖而不是主键冲突
|
|
877
|
+
const res = await milvusCall('/entities/upsert', { collectionName: collectionName(cfg), data: batch }, cfg)
|
|
878
|
+
const d = res && res.data
|
|
879
|
+
n += intOf(d && (d.upsertCount !== undefined ? d.upsertCount : d.insertCount), batch.length)
|
|
880
|
+
} catch (e) {
|
|
881
|
+
const res = await milvusCall('/entities/insert', { collectionName: collectionName(cfg), data: batch }, cfg)
|
|
882
|
+
const d = res && res.data
|
|
883
|
+
n += intOf(d && (d.insertCount !== undefined ? d.insertCount : d.insert_count), batch.length)
|
|
884
|
+
}
|
|
885
|
+
store.meta.progress = { text: '写入索引 ' + Math.min(i + batch.length, rows.length) + '/' + rows.length, done: i + batch.length, total: rows.length }
|
|
886
|
+
}
|
|
887
|
+
store.meta.progress = null
|
|
888
|
+
store.meta.collectionReady = true
|
|
889
|
+
store.meta.dimension = dim
|
|
890
|
+
return { rows: rows.length, indexed: n, dimension: dim }
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// 写入 = 先落本地库(一定成功),再尽力同步向量索引(失败只记日志,不回滚)。
|
|
894
|
+
// 这条顺序是刻意的:记忆是给人用的,不该因为外部服务没配好就写不进去。
|
|
895
|
+
async function addEntries(entries, s, source, opts) {
|
|
896
|
+
const o = opts || {}
|
|
897
|
+
const clean = []
|
|
898
|
+
for (const raw of entries) {
|
|
899
|
+
const e = asEntry(raw, source)
|
|
900
|
+
if (e) clean.push(e)
|
|
901
|
+
}
|
|
902
|
+
const skipped = entries.length - clean.length
|
|
903
|
+
if (!clean.length) return { added: 0, updated: 0, entries: 0, rows: 0, dimension: 0, indexed: 0, indexError: null, skipped: skipped }
|
|
904
|
+
|
|
905
|
+
const loc = putEntriesLocal(clean)
|
|
906
|
+
let indexed = 0, rows = 0, indexError = null
|
|
907
|
+
if (o.index !== false) {
|
|
908
|
+
// 没配向量模型 / Milvus 时**不去发请求**(否则每次写入都要等一次超时),
|
|
909
|
+
// 直接把「为什么没进索引」记下来,配好后用面板上的「同步索引」补。
|
|
910
|
+
if (!milvusBase() || !embedReady(s)) {
|
|
911
|
+
indexError = '未配置向量模型与 Milvus(本地已存住)'
|
|
912
|
+
} else {
|
|
913
|
+
try {
|
|
914
|
+
const r = await indexEntries(clean, s)
|
|
915
|
+
rows = r.rows
|
|
916
|
+
indexed = r.indexed
|
|
917
|
+
for (const e of clean) {
|
|
918
|
+
const at = indexOfEntry(e.id)
|
|
919
|
+
if (at >= 0) store.entries[at].indexed = true
|
|
920
|
+
}
|
|
921
|
+
} catch (e) {
|
|
922
|
+
indexError = msgOf(e)
|
|
923
|
+
store.meta.progress = null
|
|
924
|
+
store.meta.lastError = '向量索引同步失败(本地已存住):' + indexError
|
|
925
|
+
log('warn', store.meta.lastError)
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
recalcMeta()
|
|
930
|
+
return { added: loc.added, updated: loc.updated, ids: clean.map(function (e) { return e.id }), entries: clean.length, rows: rows, dimension: store.meta.dimension || 0, indexed: indexed, indexError: indexError, skipped: skipped }
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
// 把还没进索引(或索引失败)的条目补进 Milvus。
|
|
934
|
+
async function syncIndex(s, all) {
|
|
935
|
+
const cfg = s || settings()
|
|
936
|
+
if (!milvusBase(cfg) || !embedReady(cfg)) throw new Error('未配置 Milvus 地址或向量模型 API Key,无法同步索引')
|
|
937
|
+
const todo = all === true ? store.entries.slice() : store.entries.filter(function (e) { return e.indexed !== true })
|
|
938
|
+
if (!todo.length) return { ok: true, synced: 0, entries: 0, dimension: store.meta.dimension || 0 }
|
|
939
|
+
const r = await indexEntries(todo, cfg)
|
|
940
|
+
for (const e of todo) {
|
|
941
|
+
const at = indexOfEntry(e.id)
|
|
942
|
+
if (at >= 0) store.entries[at].indexed = true
|
|
943
|
+
}
|
|
944
|
+
recalcMeta()
|
|
945
|
+
return { ok: true, synced: r.indexed, entries: todo.length, dimension: r.dimension }
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// 删除:本地一定删掉;删索引是尽力而为 —— 连不上向量库不该让本地条目删不掉。
|
|
949
|
+
// ids 既接受条目 id,也接受检索结果里的索引行 id(`<条目 id>#…`)。
|
|
950
|
+
async function removeEntries(ids) {
|
|
951
|
+
const want = {}
|
|
952
|
+
for (const id of ids) want[String(id)] = true
|
|
953
|
+
const kept = []
|
|
954
|
+
const gone = []
|
|
955
|
+
for (const e of store.entries) {
|
|
956
|
+
if (want[e.id]) { gone.push(e); continue }
|
|
957
|
+
kept.push(e)
|
|
958
|
+
}
|
|
959
|
+
store.entries = kept
|
|
960
|
+
recalcMeta()
|
|
961
|
+
const rowIds = []
|
|
962
|
+
for (const e of gone) for (const r of chunkRowsOf(e)) rowIds.push(r.id)
|
|
963
|
+
for (const id of ids) if (String(id).indexOf('#') >= 0 && rowIds.indexOf(String(id)) < 0) rowIds.push(String(id))
|
|
964
|
+
let indexDeleted = 0, indexError = null
|
|
965
|
+
if (rowIds.length && milvusBase()) {
|
|
966
|
+
try {
|
|
967
|
+
indexDeleted = await milvusDelete(rowIds)
|
|
968
|
+
} catch (e) {
|
|
969
|
+
indexError = msgOf(e)
|
|
970
|
+
log('warn', '删除向量索引失败(本地已删除):' + indexError)
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
return { deleted: gone.length, indexDeleted: indexDeleted, indexError: indexError, gone: gone.map(function (e) { return e.id }) }
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
function hitOf(e, score, mode) { return {
|
|
977
|
+
id: e.id,
|
|
978
|
+
entryId: e.id,
|
|
979
|
+
title: e.title,
|
|
980
|
+
text: clip(e.text, 600),
|
|
981
|
+
tags: (e.tags || []).join(','),
|
|
982
|
+
kind: e.kind,
|
|
983
|
+
source: e.source,
|
|
984
|
+
created_at: e.created_at,
|
|
985
|
+
score: score,
|
|
986
|
+
rerankScore: null,
|
|
987
|
+
indexed: e.indexed === true,
|
|
988
|
+
mode: mode || 'local',
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
// 本地关键词检索:没有向量库(或索引还没同步)时的兜底,永远可用。
|
|
993
|
+
// 中文没有词边界,所以除了空白切分还要补 2 字滑窗,否则「间接注入」匹配不到「注入」。
|
|
994
|
+
function localSearch(query, k, opts) {
|
|
995
|
+
const o = opts || {}
|
|
996
|
+
const q = String(query || '').trim().toLowerCase()
|
|
997
|
+
if (!q) return []
|
|
998
|
+
const terms = []
|
|
999
|
+
for (const t of q.split(/[\s,,、;;::/|()()\[\]【】"'`]+/)) if (t) terms.push(t)
|
|
1000
|
+
for (const seg of q.replace(/[^\u4e00-\u9fa5]/g, ' ').split(/\s+/)) {
|
|
1001
|
+
for (let i = 0; i + 2 <= seg.length; i++) terms.push(seg.slice(i, i + 2))
|
|
1002
|
+
}
|
|
1003
|
+
const uniq = []
|
|
1004
|
+
for (const t of terms) if (t.length >= 2 && uniq.indexOf(t) < 0) uniq.push(t)
|
|
1005
|
+
const hits = []
|
|
1006
|
+
for (const e of store.entries) {
|
|
1007
|
+
if (typeof o.filter === 'function' && !o.filter(e)) continue
|
|
1008
|
+
const title = String(e.title || '').toLowerCase()
|
|
1009
|
+
const tags = (e.tags || []).join(',').toLowerCase()
|
|
1010
|
+
const text = String(e.text || '').toLowerCase()
|
|
1011
|
+
let score = 0, matched = 0
|
|
1012
|
+
for (const t of uniq) {
|
|
1013
|
+
let s = 0
|
|
1014
|
+
if (title.indexOf(t) >= 0) s += 3
|
|
1015
|
+
if (tags.indexOf(t) >= 0) s += 2
|
|
1016
|
+
if (text.indexOf(t) >= 0) s += 1
|
|
1017
|
+
if (s > 0) { matched++; score += s }
|
|
1018
|
+
}
|
|
1019
|
+
if (!matched) continue
|
|
1020
|
+
hits.push({ e: e, score: score + matched })
|
|
1021
|
+
}
|
|
1022
|
+
hits.sort(function (a, b) { return b.score - a.score || b.e.updated_at - a.e.updated_at })
|
|
1023
|
+
return hits.slice(0, k).map(function (h) { return hitOf(h.e, h.score, 'local') })
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
function embedReady(s) { return String((s || settings()).embed.apiKey || '').trim() !== '' }
|
|
1027
|
+
|
|
1028
|
+
async function searchKnowledge(query, topK, opts) {
|
|
1029
|
+
const s = settings()
|
|
1030
|
+
const o = opts || {}
|
|
1031
|
+
const q = String(query || '').trim()
|
|
1032
|
+
if (!q) return { ok: false, error: '检索内容为空' }
|
|
1033
|
+
const k = Math.max(1, Math.min(50, intOf(topK, s.searchTopK || 8)))
|
|
1034
|
+
|
|
1035
|
+
let rows = []
|
|
1036
|
+
let mode = 'local'
|
|
1037
|
+
let vectorError = null
|
|
1038
|
+
if (milvusBase(s) && embedReady(s)) {
|
|
1039
|
+
try {
|
|
1040
|
+
const vectors = await embedTexts([q], s)
|
|
1041
|
+
const hits = await milvusSearch(vectors[0], k, s, o.filter)
|
|
1042
|
+
rows = hits.map(function (h) {
|
|
1043
|
+
const id = String(h.id || '')
|
|
1044
|
+
return {
|
|
1045
|
+
id: id,
|
|
1046
|
+
entryId: entryIdFromRow(id),
|
|
1047
|
+
title: String(h.title || ''),
|
|
1048
|
+
text: String(h.text || ''),
|
|
1049
|
+
tags: String(h.tags || ''),
|
|
1050
|
+
kind: String(h.kind || ''),
|
|
1051
|
+
source: String(h.source || ''),
|
|
1052
|
+
score: Number(h.distance !== undefined ? h.distance : h.score) || 0,
|
|
1053
|
+
rerankScore: null,
|
|
1054
|
+
indexed: true,
|
|
1055
|
+
mode: 'vector',
|
|
1056
|
+
}
|
|
1057
|
+
})
|
|
1058
|
+
mode = 'vector'
|
|
1059
|
+
} catch (e) {
|
|
1060
|
+
vectorError = msgOf(e)
|
|
1061
|
+
log('warn', '向量检索失败,改用本地检索:' + vectorError)
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// 本地库可能比索引新(离线期间写入、索引同步失败),所以两条路都要走一遍再合并。
|
|
1066
|
+
const local = localSearch(q, k, o)
|
|
1067
|
+
const seen = {}
|
|
1068
|
+
for (const r of rows) seen[r.entryId] = true
|
|
1069
|
+
let extra = 0
|
|
1070
|
+
for (const r of local) {
|
|
1071
|
+
if (seen[r.entryId]) continue
|
|
1072
|
+
rows.push(r)
|
|
1073
|
+
seen[r.entryId] = true
|
|
1074
|
+
extra++
|
|
1075
|
+
}
|
|
1076
|
+
if (mode === 'vector') { if (extra > 0) mode = 'hybrid' } else { mode = 'local' }
|
|
1077
|
+
|
|
1078
|
+
// 召回之后再上重排:把候选文本交给重排模型重新排序
|
|
1079
|
+
const wantRerank = o.rerank === undefined ? true : o.rerank === true
|
|
1080
|
+
if (wantRerank && rerankReady(s) && rows.length > 1) {
|
|
1081
|
+
try {
|
|
1082
|
+
const order = await rerankDocs(q, rows.map(function (r) { return r.title + '\n' + r.text }), s)
|
|
1083
|
+
if (order.length) {
|
|
1084
|
+
const sorted = order.map(function (x) { const r = rows[x.index]; r.rerankScore = x.score; return r })
|
|
1085
|
+
for (const r of rows) if (r.rerankScore === null) sorted.push(r)
|
|
1086
|
+
rows.length = 0
|
|
1087
|
+
for (const r of sorted) rows.push(r)
|
|
1088
|
+
}
|
|
1089
|
+
} catch (e) {
|
|
1090
|
+
log('warn', '重排失败,返回原始召回顺序:' + msgOf(e))
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
return {
|
|
1094
|
+
ok: true, query: q, count: rows.length, mode: mode, vectorError: vectorError,
|
|
1095
|
+
indexedRows: mode === 'local' ? 0 : rows.filter(function (r) { return r.mode === 'vector' }).length,
|
|
1096
|
+
hits: rows.slice(0, k),
|
|
1097
|
+
reranked: rows.some(function (r) { return r.rerankScore !== null }),
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
// ── 状态快照(面板用) ────────────────────────────────────────────────────
|
|
1102
|
+
function snapshot() {
|
|
1103
|
+
const s = settings()
|
|
1104
|
+
return {
|
|
1105
|
+
updatedAt: store.updatedAt,
|
|
1106
|
+
settings: s,
|
|
1107
|
+
embedPresets: Object.keys(EMBED_PRESETS).map(function (k) {
|
|
1108
|
+
return { id: k, label: EMBED_PRESETS[k].label, base: EMBED_PRESETS[k].base, keyHint: EMBED_PRESETS[k].keyHint, models: EMBED_PRESETS[k].models }
|
|
1109
|
+
}),
|
|
1110
|
+
rerankPresets: Object.keys(RERANK_PRESETS).map(function (k) {
|
|
1111
|
+
return { id: k, label: RERANK_PRESETS[k].label, base: RERANK_PRESETS[k].base, models: RERANK_PRESETS[k].models }
|
|
1112
|
+
}),
|
|
1113
|
+
kinds: KINDS,
|
|
1114
|
+
status: {
|
|
1115
|
+
collection: collectionName(s),
|
|
1116
|
+
collectionReady: store.meta.collectionReady === true,
|
|
1117
|
+
dimension: store.meta.dimension || 0,
|
|
1118
|
+
rowCount: store.meta.rowCount || 0,
|
|
1119
|
+
// 本地条目是权威数据,索引条数只是它的派生视图,两个数要一起看。
|
|
1120
|
+
localCount: store.meta.localCount || 0,
|
|
1121
|
+
indexed: store.meta.indexed || 0,
|
|
1122
|
+
pending: store.meta.pending || 0,
|
|
1123
|
+
embedReady: embedReady(s),
|
|
1124
|
+
milvusReady: !!milvusBase(s),
|
|
1125
|
+
seeded: store.meta.seeded === true,
|
|
1126
|
+
persistence: store.meta.persistence || 'unknown',
|
|
1127
|
+
storePath: store.meta.storePath || '',
|
|
1128
|
+
lastError: store.meta.lastError || null,
|
|
1129
|
+
progress: store.meta.progress || null,
|
|
1130
|
+
lastOp: store.meta.lastOp || null,
|
|
1131
|
+
},
|
|
1132
|
+
importLabel: IMPORT_LABEL,
|
|
1133
|
+
importExt: IMPORT_EXT.slice(),
|
|
1134
|
+
captures: store.captures.slice(-20).reverse(),
|
|
1135
|
+
capturePhrases: capturePhrases(s),
|
|
1136
|
+
log: store.log.slice(-60),
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
async function refreshCount(s) {
|
|
1141
|
+
if (!milvusBase(s)) return -1
|
|
1142
|
+
try {
|
|
1143
|
+
const n = await milvusCount(s)
|
|
1144
|
+
if (n >= 0) store.meta.rowCount = n
|
|
1145
|
+
return n
|
|
1146
|
+
} catch (e) { return -1 }
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
// ── 对话捕获 ──────────────────────────────────────────────────────────────
|
|
1150
|
+
// 红队队员在会话里说「把这台机器的指纹写入记忆」,不该要求他改用工具。这里监听
|
|
1151
|
+
// post-commit 的 session/event;命中触发词就把那条消息收进本地库。
|
|
1152
|
+
//
|
|
1153
|
+
// 三个刻意的选择:
|
|
1154
|
+
// 1. 只认用户消息。模型回复里的「记忆」多半是在解释功能本身,捕进来是噪声。
|
|
1155
|
+
// 2. 入库不依赖向量模型 —— 先落本地库,索引同步失败只记日志(捕获必须「说了就记住」)。
|
|
1156
|
+
// 3. 同一个会话序号只捕一次,事件重放/重复投递不会写出两条。
|
|
1157
|
+
const captured = {}
|
|
1158
|
+
let captureBusy = false
|
|
1159
|
+
|
|
1160
|
+
function textOfContent(content) {
|
|
1161
|
+
if (typeof content === 'string') return content
|
|
1162
|
+
if (!Array.isArray(content)) return ''
|
|
1163
|
+
const parts = []
|
|
1164
|
+
for (const b of content) {
|
|
1165
|
+
if (!b || typeof b !== 'object') continue
|
|
1166
|
+
if (b.type === 'text' && typeof b.text === 'string') parts.push(b.text)
|
|
1167
|
+
}
|
|
1168
|
+
return parts.join('\n')
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
function matchCapturePhrase(text) {
|
|
1172
|
+
const lower = String(text || '').toLowerCase()
|
|
1173
|
+
for (const p of capturePhrases()) {
|
|
1174
|
+
const at = lower.indexOf(p.toLowerCase())
|
|
1175
|
+
if (at >= 0) return { phrase: p, at: at }
|
|
1176
|
+
}
|
|
1177
|
+
return null
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
// 「请把这条写入记忆:xxx」-> 「xxx」。去掉触发词与它两侧的连接词,剩下的才是要记的内容。
|
|
1181
|
+
function captureBody(text, hit) {
|
|
1182
|
+
const raw = String(text || '')
|
|
1183
|
+
let head = raw.slice(0, hit.at)
|
|
1184
|
+
let tail = raw.slice(hit.at + hit.phrase.length)
|
|
1185
|
+
tail = tail.replace(/^[\s::,,、。.;;\-—]*/, '')
|
|
1186
|
+
head = head.replace(/[\s::,,、。.;;\-—]*$/, '')
|
|
1187
|
+
head = head.replace(/^(请|麻烦|帮我|帮忙|记得|记住|把|将|这条|这段|这个|这条信息)+/g, '')
|
|
1188
|
+
const body = (head && tail) ? (head + '\n' + tail) : (head || tail)
|
|
1189
|
+
return body.trim() || raw.trim()
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
function sessionTitleOf(session) {
|
|
1193
|
+
const svc = ctx.get('sessionTitle')
|
|
1194
|
+
try {
|
|
1195
|
+
if (svc && typeof svc.get === 'function') {
|
|
1196
|
+
const t = svc.get(session)
|
|
1197
|
+
const v = t && (t.title || t.value || t.text)
|
|
1198
|
+
if (v) return clip(v, 80)
|
|
1199
|
+
}
|
|
1200
|
+
} catch (e) { /* 标题只为人看着方便,取不到就退回会话 id */ }
|
|
1201
|
+
return ''
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
// 触发词后面没跟内容时(「记住这个」),取上一条消息当上下文 —— 否则记下来的是空壳。
|
|
1205
|
+
function previousTextOf(session, seq) {
|
|
1206
|
+
let events = []
|
|
1207
|
+
try { events = session.snapshotEvents() } catch (e) { return '' }
|
|
1208
|
+
if (!Array.isArray(events)) return ''
|
|
1209
|
+
let best = ''
|
|
1210
|
+
for (const ev of events) {
|
|
1211
|
+
if (!ev || Number(ev.seq) >= Number(seq)) continue
|
|
1212
|
+
if (ev.type === 'user/message') best = textOfContent(ev.data && ev.data.content)
|
|
1213
|
+
else if (ev.type === 'assistant/message') {
|
|
1214
|
+
const t = textOfContent(ev.data && ev.data.message && ev.data.message.content)
|
|
1215
|
+
if (t.trim()) best = t
|
|
1216
|
+
}
|
|
1217
|
+
if (best.length > 4000) best = best.slice(0, 4000)
|
|
1218
|
+
}
|
|
1219
|
+
return clip(best, 1200)
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
async function captureFrom(session, event) {
|
|
1223
|
+
const text = textOfContent(event.data && event.data.content)
|
|
1224
|
+
if (!text.trim()) return null
|
|
1225
|
+
const hit = matchCapturePhrase(text)
|
|
1226
|
+
if (!hit) return null
|
|
1227
|
+
const key = String((session && session.id) || '') + '#' + String(event.seq)
|
|
1228
|
+
if (captured[key]) return null
|
|
1229
|
+
captured[key] = true
|
|
1230
|
+
|
|
1231
|
+
const sid = String((session && session.id) || '')
|
|
1232
|
+
const title = session ? sessionTitleOf(session) : ''
|
|
1233
|
+
const body = captureBody(text, hit)
|
|
1234
|
+
let finalText = body
|
|
1235
|
+
if (settings().capture.withContext && body.replace(/\s+/g, '').length < 12 && session) {
|
|
1236
|
+
const around = previousTextOf(session, event.seq)
|
|
1237
|
+
if (around) finalText = around + '\n\n(对话里出现触发词「' + hit.phrase + '」,随上下文一并记下)'
|
|
1238
|
+
}
|
|
1239
|
+
const label = '对话捕获 · ' + (title || sid.slice(0, 12) || '面板')
|
|
1240
|
+
const r = await addEntries([{
|
|
1241
|
+
title: clip(body.split('\n')[0] || body, 60),
|
|
1242
|
+
text: finalText,
|
|
1243
|
+
kind: settings().capture.kind,
|
|
1244
|
+
tags: ['对话捕获'],
|
|
1245
|
+
source: label,
|
|
1246
|
+
}], null, label)
|
|
1247
|
+
|
|
1248
|
+
store.captures.push({
|
|
1249
|
+
at: nowMs(), session: sid, title: title || sid.slice(0, 12), phrase: hit.phrase,
|
|
1250
|
+
entryId: (r.ids && r.ids[0]) || '', text: clip(finalText, 200),
|
|
1251
|
+
indexed: r.indexed > 0, error: r.indexError || null,
|
|
1252
|
+
})
|
|
1253
|
+
if (store.captures.length > CAPTURE_MAX) store.captures = store.captures.slice(-CAPTURE_MAX)
|
|
1254
|
+
log('ok', '对话捕获(' + hit.phrase + '):' + clip(finalText, 100) + (r.indexError ? '[索引未同步,本地已存]' : ''))
|
|
1255
|
+
store.meta.lastOp = { at: nowMs(), op: 'capture', text: clip(finalText, 120) }
|
|
1256
|
+
await persist()
|
|
1257
|
+
return r
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
ctx.effect(function () {
|
|
1261
|
+
const off = ctx.on('session/event', function (session, event) {
|
|
1262
|
+
if (!event || event.type !== 'user/message') return
|
|
1263
|
+
if (!settings().capture.enabled) return
|
|
1264
|
+
// 捕获要写库 + 落盘(读改写),串行化,避免并发互相覆盖。
|
|
1265
|
+
if (captureBusy) return
|
|
1266
|
+
captureBusy = true
|
|
1267
|
+
Promise.resolve()
|
|
1268
|
+
.then(function () { return ensureLoaded() })
|
|
1269
|
+
.then(function () { return captureFrom(session, event) })
|
|
1270
|
+
.catch(function (e) { console.error('[rtmemory] 对话捕获失败: ' + msgOf(e)) })
|
|
1271
|
+
.then(function () { captureBusy = false })
|
|
1272
|
+
})
|
|
1273
|
+
return function () { if (typeof off === 'function') off() }
|
|
1274
|
+
}, 'redteam-memory: 对话捕获')
|
|
1275
|
+
|
|
1276
|
+
// 对外服务:其它插件(例如报告插件把成稿导入记忆)不必各自去读写存储文件。
|
|
1277
|
+
ctx.effect(function () {
|
|
1278
|
+
if (typeof ctx.provide !== 'function') {
|
|
1279
|
+
log('warn', '当前运行时不支持 ctx.provide,跳过 redteamMemory 服务注册')
|
|
1280
|
+
return function () {}
|
|
1281
|
+
}
|
|
1282
|
+
return ctx.provide('redteamMemory', {
|
|
1283
|
+
add: async function (entries, source) {
|
|
1284
|
+
await ensureLoaded()
|
|
1285
|
+
const list = Array.isArray(entries) ? entries : [entries]
|
|
1286
|
+
const r = await addEntries(list, null, source || 'plugin')
|
|
1287
|
+
log('ok', '外部插件写入记忆 ' + r.added + ' 条(来源 ' + (source || 'plugin') + ',本地共 ' + (store.meta.localCount || 0) + ' 条)')
|
|
1288
|
+
await persist()
|
|
1289
|
+
return {
|
|
1290
|
+
added: r.added, updated: r.updated, entries: r.entries, rows: r.rows, dimension: r.dimension,
|
|
1291
|
+
indexed: r.indexed, indexError: r.indexError, localCount: store.meta.localCount || 0, ids: r.ids || [],
|
|
1292
|
+
}
|
|
1293
|
+
},
|
|
1294
|
+
search: async function (query, topK, opts) {
|
|
1295
|
+
await ensureLoaded()
|
|
1296
|
+
return await searchKnowledge(query, topK, opts)
|
|
1297
|
+
},
|
|
1298
|
+
stats: async function () {
|
|
1299
|
+
await ensureLoaded()
|
|
1300
|
+
recalcMeta()
|
|
1301
|
+
return {
|
|
1302
|
+
localCount: store.meta.localCount || 0, indexed: store.meta.indexed || 0,
|
|
1303
|
+
pending: store.meta.pending || 0, persistence: store.meta.persistence || 'unknown',
|
|
1304
|
+
storePath: store.meta.storePath || '',
|
|
1305
|
+
}
|
|
1306
|
+
},
|
|
1307
|
+
})
|
|
1308
|
+
}, 'redteam-memory: 对外服务 redteamMemory')
|
|
1309
|
+
|
|
1310
|
+
// ── 模型工具 ──────────────────────────────────────────────────────────────
|
|
1311
|
+
const searchTool = harness.defineTool({
|
|
1312
|
+
name: 'memory_search',
|
|
1313
|
+
description: '在红队记忆库里检索 AI 安全知识与进攻技巧(OWASP LLM Top 10、提示词注入、RAG 投毒、Agent/MCP 工具滥用、推理服务未授权与已知 CVE、证据与合规清单等)。做 AI/LLM 方向测试、需要判断某个手法或风险点、或写报告时用这个工具,比凭印象回答可靠。',
|
|
1314
|
+
parameters: {
|
|
1315
|
+
type: 'object',
|
|
1316
|
+
properties: {
|
|
1317
|
+
query: { type: 'string', description: '要检索的内容,自然语言即可,例如「间接提示词注入怎么验证」' },
|
|
1318
|
+
topK: { type: 'integer', description: '返回条数,默认取设置里的值' },
|
|
1319
|
+
rerank: { type: 'boolean', description: '是否用重排模型精排(默认按设置里的开关)' },
|
|
1320
|
+
},
|
|
1321
|
+
required: ['query'],
|
|
1322
|
+
},
|
|
1323
|
+
output: {
|
|
1324
|
+
schema: { type: 'json' },
|
|
1325
|
+
render: function (args, value) {
|
|
1326
|
+
if (!value || value.ok !== true) return [{ type: 'text', text: '检索失败:' + ((value && value.error) || '未知错误') }]
|
|
1327
|
+
if (!value.hits || value.hits.length === 0) return [{ type: 'text', text: '记忆库里没有匹配内容(检索:' + value.query + ')。可以先用 memory_add 导入相关知识。' }]
|
|
1328
|
+
const how = value.mode === 'vector' ? '向量检索' : (value.mode === 'hybrid' ? '向量 + 本地关键词' : '本地关键词检索')
|
|
1329
|
+
const lines = ['命中 ' + value.hits.length + ' 条(' + how + ')' + (value.reranked ? ',已重排' : '') + ':']
|
|
1330
|
+
if (value.vectorError) lines.push('(向量检索失败,已退回本地:' + value.vectorError + ')')
|
|
1331
|
+
for (const h of value.hits) {
|
|
1332
|
+
lines.push('')
|
|
1333
|
+
lines.push('· ' + h.title + ' [' + h.kind + (h.tags ? ' / ' + h.tags : '') + '] 分数 ' + Number(h.score).toFixed(3))
|
|
1334
|
+
lines.push(' ' + h.text)
|
|
1335
|
+
}
|
|
1336
|
+
return [{ type: 'text', text: lines.join('\n') }]
|
|
1337
|
+
},
|
|
1338
|
+
},
|
|
1339
|
+
execute: async function (args) {
|
|
1340
|
+
await ensureLoaded()
|
|
1341
|
+
const r = await searchKnowledge(args && args.query, args && args.topK, { rerank: args && args.rerank })
|
|
1342
|
+
store.meta.lastOp = { at: nowMs(), op: 'memory_search', text: clip((args && args.query) || '', 120) }
|
|
1343
|
+
await persist()
|
|
1344
|
+
return r
|
|
1345
|
+
},
|
|
1346
|
+
})
|
|
1347
|
+
|
|
1348
|
+
const addTool = harness.defineTool({
|
|
1349
|
+
name: 'memory_add',
|
|
1350
|
+
description: '往红队记忆库写入一条知识(AI 安全知识、进攻技巧、检查清单、复盘结论)。先落本地库(一定成功),再尽力同步向量索引 —— 就算没配向量模型,内容也能存住并被 memory_search 检索到。',
|
|
1351
|
+
parameters: {
|
|
1352
|
+
type: 'object',
|
|
1353
|
+
properties: {
|
|
1354
|
+
text: { type: 'string', description: '知识正文。一段一个主题,不要写成流水账。' },
|
|
1355
|
+
title: { type: 'string', description: '标题(省略时取正文首行)' },
|
|
1356
|
+
kind: { type: 'string', description: 'knowledge / technique / payload / checklist / note' },
|
|
1357
|
+
tags: { type: 'array', items: { type: 'string' }, description: '标签,例如 owasp、注入、rag、agent' },
|
|
1358
|
+
source: { type: 'string', description: '来源:CVE 编号、文档链接、某次测试的复盘' },
|
|
1359
|
+
},
|
|
1360
|
+
required: ['text'],
|
|
1361
|
+
},
|
|
1362
|
+
output: {
|
|
1363
|
+
schema: { type: 'json' },
|
|
1364
|
+
render: function (args, value) {
|
|
1365
|
+
if (!value || value.ok !== true) return [{ type: 'text', text: '写入失败:' + ((value && value.error) || '未知错误') }]
|
|
1366
|
+
const head = '已写入本地记忆库 ' + value.added + ' 条(本地共 ' + (value.localCount || 0) + ' 条)'
|
|
1367
|
+
const tail = value.indexError
|
|
1368
|
+
? ';向量索引未同步(' + value.indexError + '),本地检索仍可用,配好向量模型后在面板点「同步索引」即可。'
|
|
1369
|
+
: ';向量索引已同步 ' + (value.indexed || 0) + ' 行(维度 ' + (value.dimension || '?') + ')。'
|
|
1370
|
+
return [{ type: 'text', text: head + tail }]
|
|
1371
|
+
},
|
|
1372
|
+
},
|
|
1373
|
+
execute: async function (args) {
|
|
1374
|
+
await ensureLoaded()
|
|
1375
|
+
const r = await addEntries([args], null, 'model')
|
|
1376
|
+
store.meta.lastOp = { at: nowMs(), op: 'memory_add', text: clip((args && (args.title || args.text)) || '', 120) }
|
|
1377
|
+
log('ok', '模型写入记忆 ' + r.added + ' 条(本地 ' + (store.meta.localCount || 0) + ' 条,索引 ' + r.indexed + ' 行' + (r.indexError ? ',索引失败' : '') + '):' + clip((args && (args.title || args.text)) || '', 100))
|
|
1378
|
+
await refreshCount()
|
|
1379
|
+
await persist()
|
|
1380
|
+
return Object.assign({ ok: true, localCount: store.meta.localCount || 0 }, r)
|
|
1381
|
+
},
|
|
1382
|
+
})
|
|
1383
|
+
|
|
1384
|
+
const deleteTool = harness.defineTool({
|
|
1385
|
+
name: 'memory_delete',
|
|
1386
|
+
description: '从红队记忆库按 id 删除条目。id 从 memory_search 的结果里拿。删除不可恢复,删前先确认 id。',
|
|
1387
|
+
parameters: {
|
|
1388
|
+
type: 'object',
|
|
1389
|
+
properties: { ids: { type: 'array', items: { type: 'string' }, description: '要删除的条目 id 列表' } },
|
|
1390
|
+
required: ['ids'],
|
|
1391
|
+
},
|
|
1392
|
+
output: {
|
|
1393
|
+
schema: { type: 'json' },
|
|
1394
|
+
render: function (args, value) {
|
|
1395
|
+
if (!value || value.ok !== true) return [{ type: 'text', text: '删除失败:' + ((value && value.error) || '未知错误') }]
|
|
1396
|
+
return [{ type: 'text', text: '已删除 ' + value.deleted + ' 条' + (value.indexError ? '(向量索引未同步删除:' + value.indexError + ')' : '') }]
|
|
1397
|
+
},
|
|
1398
|
+
},
|
|
1399
|
+
execute: async function (args) {
|
|
1400
|
+
await ensureLoaded()
|
|
1401
|
+
const ids = Array.isArray(args && args.ids) ? args.ids.map(String).filter(Boolean) : []
|
|
1402
|
+
if (!ids.length) return { ok: false, error: 'ids 为空' }
|
|
1403
|
+
const r = await removeEntries(ids)
|
|
1404
|
+
log('warn', '模型删除记忆 ' + r.deleted + ' 条:' + r.gone.slice(0, 5).join(', '))
|
|
1405
|
+
store.meta.lastOp = { at: nowMs(), op: 'memory_delete', text: ids.length + ' 条' }
|
|
1406
|
+
await refreshCount()
|
|
1407
|
+
await persist()
|
|
1408
|
+
return { ok: true, deleted: r.deleted, indexDeleted: r.indexDeleted, indexError: r.indexError }
|
|
1409
|
+
},
|
|
1410
|
+
})
|
|
1411
|
+
|
|
1412
|
+
for (const t of [searchTool, addTool, deleteTool]) harness.registerTool(ctx, t)
|
|
1413
|
+
|
|
1414
|
+
// ── RPC 句柄(客户端 host.call 调) ───────────────────────────────────────
|
|
1415
|
+
harness.handle('snapshot', async function () {
|
|
1416
|
+
await ensureLoaded()
|
|
1417
|
+
if (milvusBase()) { try { await refreshCount() } catch (e) { /* 连不上时保留旧值 */ } }
|
|
1418
|
+
return { ok: true, snapshot: snapshot() }
|
|
1419
|
+
})
|
|
1420
|
+
|
|
1421
|
+
harness.handle('saveSettings', async function (args) {
|
|
1422
|
+
await ensureLoaded()
|
|
1423
|
+
mergeSettings(args && typeof args === 'object' ? args : {})
|
|
1424
|
+
const cap = settings().capture
|
|
1425
|
+
log('info', '设置已保存(Milvus ' + (milvusBase() || '未配置') + ',集合 ' + collectionName() + ',向量模型 ' + settings().embed.model
|
|
1426
|
+
+ ';对话捕获 ' + (cap.enabled ? '开' : '关') + ',触发词 ' + capturePhrases().length + ' 个)')
|
|
1427
|
+
await persist()
|
|
1428
|
+
return { ok: true, snapshot: snapshot() }
|
|
1429
|
+
})
|
|
1430
|
+
|
|
1431
|
+
// 触发词干跑:判断「这句话会不会被捕获」,不写入任何东西 —— 调触发词时用它验。
|
|
1432
|
+
harness.handle('captureTest', async function (args) {
|
|
1433
|
+
await ensureLoaded()
|
|
1434
|
+
const text = String((args && args.text) || '')
|
|
1435
|
+
if (!text.trim()) return { ok: false, error: '先给一句话再试' }
|
|
1436
|
+
const hit = matchCapturePhrase(text)
|
|
1437
|
+
if (!hit) return { ok: true, matched: false, phrases: capturePhrases(), hint: '没有一个触发词命中,这句话不会被捕获' }
|
|
1438
|
+
return { ok: true, matched: true, phrase: hit.phrase, body: captureBody(text, hit), kind: settings().capture.kind, text: clip(captureBody(text, hit), 400) }
|
|
1439
|
+
})
|
|
1440
|
+
|
|
1441
|
+
// 手工捕获:把面板里的一段文字直接收进记忆(不走会话事件,路径与自动捕获一致)。
|
|
1442
|
+
harness.handle('captureAdd', async function (args) {
|
|
1443
|
+
await ensureLoaded()
|
|
1444
|
+
const text = String((args && args.text) || '').trim()
|
|
1445
|
+
if (!text) return { ok: false, error: '内容为空' }
|
|
1446
|
+
const r = await addEntries([{
|
|
1447
|
+
title: clip(String((args && args.title) || text.split('\n')[0]), 60),
|
|
1448
|
+
text: text,
|
|
1449
|
+
kind: KINDS.indexOf(String(args && args.kind)) >= 0 ? String(args.kind) : settings().capture.kind,
|
|
1450
|
+
tags: ['对话捕获'],
|
|
1451
|
+
source: '人工捕获',
|
|
1452
|
+
}], null, '人工捕获')
|
|
1453
|
+
store.captures.push({ at: nowMs(), session: '', title: '人工捕获', phrase: '', entryId: (r.ids && r.ids[0]) || '', text: clip(text, 200), indexed: r.indexed > 0, error: r.indexError || null })
|
|
1454
|
+
if (store.captures.length > CAPTURE_MAX) store.captures = store.captures.slice(-CAPTURE_MAX)
|
|
1455
|
+
log('ok', '人工捕获一条:' + clip(text, 100))
|
|
1456
|
+
await persist()
|
|
1457
|
+
return { ok: true, snapshot: snapshot(), added: r.added, updated: r.updated, entries: r.entries, rows: r.rows, dimension: r.dimension, indexed: r.indexed, indexError: r.indexError }
|
|
1458
|
+
})
|
|
1459
|
+
|
|
1460
|
+
harness.handle('captureClear', async function () {
|
|
1461
|
+
store.captures = []
|
|
1462
|
+
await persist()
|
|
1463
|
+
return { ok: true, snapshot: snapshot() }
|
|
1464
|
+
})
|
|
1465
|
+
|
|
1466
|
+
// Connection/configuration failures are business results, not failed RPC handlers.
|
|
1467
|
+
function connectionTestError(error) {
|
|
1468
|
+
const message = msgOf(error)
|
|
1469
|
+
const hint = /未配置|未启用|未选模型|未填/.test(message)
|
|
1470
|
+
? '。请在「设置 → 红队设置」填写对应配置,先点击「保存记忆设置」,再测试;测试使用已保存配置。' : ''
|
|
1471
|
+
return { ok: false, error: message + hint }
|
|
1472
|
+
}
|
|
1473
|
+
function handleConnectionTest(method, test) {
|
|
1474
|
+
harness.handle(method, async function (args) {
|
|
1475
|
+
try {
|
|
1476
|
+
const result = await test(args)
|
|
1477
|
+
return result && result.ok === false ? connectionTestError(result.error || '连接测试失败') : result
|
|
1478
|
+
} catch (error) { return connectionTestError(error) }
|
|
1479
|
+
})
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
handleConnectionTest('testMilvus', async function () {
|
|
1483
|
+
await ensureLoaded()
|
|
1484
|
+
const t0 = nowMs()
|
|
1485
|
+
const list = await milvusListCollections()
|
|
1486
|
+
const has = list.indexOf(collectionName()) >= 0
|
|
1487
|
+
if (has) {
|
|
1488
|
+
const n = await milvusCount()
|
|
1489
|
+
if (n >= 0) store.meta.rowCount = n
|
|
1490
|
+
store.meta.collectionReady = true
|
|
1491
|
+
}
|
|
1492
|
+
log('ok', 'Milvus 连通:共 ' + list.length + ' 个 collection' + (has ? ',其中 ' + collectionName() + ' 存在(' + (store.meta.rowCount || 0) + ' 行)' : ',目标 collection 尚不存在(首次写入时自动创建)'))
|
|
1493
|
+
await persist()
|
|
1494
|
+
return { ok: true, collections: list, collection: collectionName(), exists: has, rowCount: store.meta.rowCount || 0, ms: nowMs() - t0 }
|
|
1495
|
+
})
|
|
1496
|
+
|
|
1497
|
+
handleConnectionTest('testEmbed', async function () {
|
|
1498
|
+
await ensureLoaded()
|
|
1499
|
+
// Custom OpenAI-compatible endpoints may intentionally allow anonymous access.
|
|
1500
|
+
if (!embedReady() && settings().embed.provider !== 'openai') return { ok: false, error: '未配置向量模型 API Key' }
|
|
1501
|
+
const t0 = nowMs()
|
|
1502
|
+
const vecs = await embedTexts(['连接测试:红队记忆库'], null)
|
|
1503
|
+
const dim = vecs[0].length
|
|
1504
|
+
log('ok', '向量模型连通:' + settings().embed.model + ',维度 ' + dim)
|
|
1505
|
+
return { ok: true, model: settings().embed.model, dimension: dim, ms: nowMs() - t0, head: vecs[0].slice(0, 6) }
|
|
1506
|
+
})
|
|
1507
|
+
|
|
1508
|
+
handleConnectionTest('testRerank', async function () {
|
|
1509
|
+
await ensureLoaded()
|
|
1510
|
+
if (!rerankReady()) return { ok: false, error: '重排未启用或未选模型' }
|
|
1511
|
+
const docs = ['提示词注入:把指令伪装成数据塞进上下文', '模型窃取:通过大量查询近似复制模型', '端口扫描:枚举目标开放端口']
|
|
1512
|
+
const t0 = nowMs()
|
|
1513
|
+
const r = await rerankDocs('怎么做提示词注入测试', docs, null)
|
|
1514
|
+
log('ok', '重排模型连通:' + settings().rerank.model + ',返回 ' + r.length + ' 条排序')
|
|
1515
|
+
return { ok: true, model: settings().rerank.model, order: r, ms: nowMs() - t0 }
|
|
1516
|
+
})
|
|
1517
|
+
|
|
1518
|
+
handleConnectionTest('testS3', async function () {
|
|
1519
|
+
await ensureLoaded()
|
|
1520
|
+
if (!s3Cfg().enabled) return { ok: false, error: 'MinIO 未启用(先在上面勾选并填端点/桶)' }
|
|
1521
|
+
const xml = await s3Call('GET', '', 'list-type=2&max-keys=1', null, null)
|
|
1522
|
+
const parsed = parseS3List(xml)
|
|
1523
|
+
log('ok', 'MinIO 连通:桶 ' + s3Cfg().bucket + ' 可列举')
|
|
1524
|
+
return { ok: true, bucket: s3Cfg().bucket, sample: parsed.objects.length }
|
|
1525
|
+
})
|
|
1526
|
+
|
|
1527
|
+
harness.handle('s3List', async function (args) {
|
|
1528
|
+
await ensureLoaded()
|
|
1529
|
+
if (!s3Cfg().enabled) return { ok: false, error: 'MinIO 未启用' }
|
|
1530
|
+
const r = await s3List(args && args.prefix, args && args.limit)
|
|
1531
|
+
return { ok: true, bucket: s3Cfg().bucket, objects: r.objects, truncated: r.truncated, nextToken: r.nextToken }
|
|
1532
|
+
})
|
|
1533
|
+
|
|
1534
|
+
// 把整库导出成 JSON 存进 MinIO(Milvus 的存储桶,例如 mimo)—— 备份 / 交付用。
|
|
1535
|
+
// 导出的源是**本地库**:本地才是权威数据,索引可能还没同步完。
|
|
1536
|
+
harness.handle('s3Backup', async function (args) {
|
|
1537
|
+
await ensureLoaded()
|
|
1538
|
+
if (!s3Ready()) return { ok: false, error: 'MinIO 未启用或未填端点/桶' }
|
|
1539
|
+
const rows = store.entries.map(function (e) {
|
|
1540
|
+
return {
|
|
1541
|
+
id: e.id, title: e.title, text: e.text, tags: (e.tags || []).join(','), kind: e.kind,
|
|
1542
|
+
source: e.source, created_at: e.created_at, updated_at: e.updated_at, indexed: e.indexed === true,
|
|
1543
|
+
}
|
|
1544
|
+
})
|
|
1545
|
+
const key = String((args && args.key) || '').trim() || ('memory-backup-' + new Date().toISOString().slice(0, 19).replace(/[:T]/g, '-') + '.json')
|
|
1546
|
+
const payload = JSON.stringify({ exportedAt: new Date().toISOString(), collection: collectionName(), count: rows.length, entries: rows }, null, 2)
|
|
1547
|
+
await s3Call('PUT', key, '', payload, null, { contentType: 'application/json' })
|
|
1548
|
+
log('ok', '已导出 ' + rows.length + ' 条到 s3://' + s3Cfg().bucket + '/' + key)
|
|
1549
|
+
await persist()
|
|
1550
|
+
return { ok: true, key: key, count: rows.length, bucket: s3Cfg().bucket }
|
|
1551
|
+
})
|
|
1552
|
+
|
|
1553
|
+
harness.handle('listKnowledge', async function (args) {
|
|
1554
|
+
await ensureLoaded()
|
|
1555
|
+
// 列表直接读本地库:它才是权威数据,也就不会出现「索引没同步就看不到」的空列表。
|
|
1556
|
+
const limit = Math.max(1, Math.min(500, intOf(args && args.limit, LIST_DEFAULT)))
|
|
1557
|
+
const offset = Math.max(0, intOf(args && args.offset, 0))
|
|
1558
|
+
const kw = String((args && args.keyword) || '').trim().toLowerCase()
|
|
1559
|
+
const kind = String((args && args.kind) || '').trim()
|
|
1560
|
+
let list = store.entries.slice().sort(function (a, b) { return b.updated_at - a.updated_at })
|
|
1561
|
+
if (kind) list = list.filter(function (e) { return e.kind === kind })
|
|
1562
|
+
if (kw) {
|
|
1563
|
+
list = list.filter(function (e) {
|
|
1564
|
+
return String(e.title || '').toLowerCase().indexOf(kw) >= 0
|
|
1565
|
+
|| String(e.text || '').toLowerCase().indexOf(kw) >= 0
|
|
1566
|
+
|| (e.tags || []).join(',').toLowerCase().indexOf(kw) >= 0
|
|
1567
|
+
|| String(e.source || '').toLowerCase().indexOf(kw) >= 0
|
|
1568
|
+
})
|
|
1569
|
+
}
|
|
1570
|
+
const total = list.length
|
|
1571
|
+
const page = list.slice(offset, offset + limit).map(function (e) {
|
|
1572
|
+
return {
|
|
1573
|
+
id: e.id, title: e.title, text: clip(e.text, 160), tags: (e.tags || []).join(','),
|
|
1574
|
+
kind: e.kind, source: e.source, created_at: e.created_at, updated_at: e.updated_at,
|
|
1575
|
+
indexed: e.indexed === true, chars: String(e.text || '').length,
|
|
1576
|
+
}
|
|
1577
|
+
})
|
|
1578
|
+
return { ok: true, entries: page, total: total, limit: limit, offset: offset, local: store.entries.length }
|
|
1579
|
+
})
|
|
1580
|
+
|
|
1581
|
+
harness.handle('getEntry', async function (args) {
|
|
1582
|
+
await ensureLoaded()
|
|
1583
|
+
const id = String((args && args.id) || '').replace(/"/g, '')
|
|
1584
|
+
if (!id) return { ok: false, error: '缺少 id' }
|
|
1585
|
+
const at = indexOfEntry(id)
|
|
1586
|
+
if (at < 0) return { ok: false, error: '找不到该条目(可能刚被删除)' }
|
|
1587
|
+
const e = store.entries[at]
|
|
1588
|
+
return { ok: true, entry: Object.assign({}, e, { tags: (e.tags || []).join(',') }) }
|
|
1589
|
+
})
|
|
1590
|
+
|
|
1591
|
+
harness.handle('search', async function (args) {
|
|
1592
|
+
await ensureLoaded()
|
|
1593
|
+
const r = await searchKnowledge(args && args.query, args && args.topK, { rerank: args && args.rerank })
|
|
1594
|
+
store.meta.lastOp = { at: nowMs(), op: 'search', text: clip((args && args.query) || '', 120) }
|
|
1595
|
+
return r
|
|
1596
|
+
})
|
|
1597
|
+
|
|
1598
|
+
// 把本地还没进索引的条目补进 Milvus(all=true 时整库重建索引)。
|
|
1599
|
+
harness.handle('syncIndex', async function (args) {
|
|
1600
|
+
await ensureLoaded()
|
|
1601
|
+
const all = boolOf(args && args.all, false)
|
|
1602
|
+
try {
|
|
1603
|
+
const r = await syncIndex(null, all)
|
|
1604
|
+
if (r.entries > 0) log('ok', '索引同步完成:' + r.entries + ' 条本地条目 -> ' + r.synced + ' 行(维度 ' + (r.dimension || '?') + ')')
|
|
1605
|
+
else log('info', '索引已是最新,无需同步')
|
|
1606
|
+
store.meta.lastOp = { at: nowMs(), op: 'syncIndex', text: r.entries + ' 条' }
|
|
1607
|
+
store.meta.lastError = null
|
|
1608
|
+
await refreshCount()
|
|
1609
|
+
await persist()
|
|
1610
|
+
return { ok: true, snapshot: snapshot(), entries: r.entries, synced: r.synced }
|
|
1611
|
+
} catch (e) {
|
|
1612
|
+
store.meta.progress = null
|
|
1613
|
+
logFailure('索引同步失败', e, 'syncIndex')
|
|
1614
|
+
await persist()
|
|
1615
|
+
return { ok: false, error: msgOf(e), snapshot: snapshot() }
|
|
1616
|
+
}
|
|
1617
|
+
})
|
|
1618
|
+
|
|
1619
|
+
// 从文件导入:pdf / word(.docx) / md / txt
|
|
1620
|
+
harness.handle('importFile', async function (args) {
|
|
1621
|
+
await ensureLoaded()
|
|
1622
|
+
const path = String((args && args.path) || '').trim()
|
|
1623
|
+
if (!path) return { ok: false, error: '缺少文件路径(用绝对路径)', snapshot: snapshot() }
|
|
1624
|
+
try {
|
|
1625
|
+
const doc = await readImport(path)
|
|
1626
|
+
const entries = entriesFromDoc(path, doc)
|
|
1627
|
+
if (!entries.length) return { ok: false, error: '从文件里没抽到可入库的文字:' + path, snapshot: snapshot() }
|
|
1628
|
+
const r = await addEntries(entries, null, path)
|
|
1629
|
+
log('ok', '导入 ' + clip(path, 90) + '(' + doc.ext + ',' + doc.how + ',' + doc.text.length + ' 字)→ '
|
|
1630
|
+
+ r.added + ' 条' + (r.indexError ? '(本地已存,索引未同步)' : '(索引 ' + r.indexed + ' 行)'))
|
|
1631
|
+
store.meta.lastOp = { at: nowMs(), op: 'importFile', text: clip(path, 120) }
|
|
1632
|
+
await refreshCount()
|
|
1633
|
+
await persist()
|
|
1634
|
+
return {
|
|
1635
|
+
ok: true, snapshot: snapshot(), path: path, ext: doc.ext, how: doc.how,
|
|
1636
|
+
chars: doc.text.length, added: r.added, updated: r.updated, parsed: entries.length,
|
|
1637
|
+
indexed: r.indexed, indexError: r.indexError,
|
|
1638
|
+
}
|
|
1639
|
+
} catch (e) {
|
|
1640
|
+
store.meta.progress = null
|
|
1641
|
+
logFailure('导入失败', e, clip(path, 80))
|
|
1642
|
+
await persist()
|
|
1643
|
+
return { ok: false, error: msgOf(e), snapshot: snapshot() }
|
|
1644
|
+
}
|
|
1645
|
+
})
|
|
1646
|
+
|
|
1647
|
+
harness.handle('seed', async function () {
|
|
1648
|
+
await ensureLoaded()
|
|
1649
|
+
try {
|
|
1650
|
+
const r = await addEntries(SEED_KNOWLEDGE, null, 'builtin')
|
|
1651
|
+
store.meta.seeded = true
|
|
1652
|
+
log('ok', '已导入内置知识包 ' + r.added + ' 条(AI 安全知识与进攻技巧),索引 ' + r.indexed + ' 行' + (r.indexError ? '(索引未同步,本地已存)' : ''))
|
|
1653
|
+
store.meta.lastOp = { at: nowMs(), op: 'seed', text: r.added + ' 条' }
|
|
1654
|
+
await refreshCount()
|
|
1655
|
+
await persist()
|
|
1656
|
+
return { ok: true, snapshot: snapshot(), added: r.added }
|
|
1657
|
+
} catch (e) {
|
|
1658
|
+
store.meta.progress = null
|
|
1659
|
+
logFailure('内置知识包导入失败', e, 'seed')
|
|
1660
|
+
await persist()
|
|
1661
|
+
return { ok: false, error: msgOf(e), snapshot: snapshot() }
|
|
1662
|
+
}
|
|
1663
|
+
})
|
|
1664
|
+
|
|
1665
|
+
harness.handle('removeKnowledge', async function (args) {
|
|
1666
|
+
await ensureLoaded()
|
|
1667
|
+
const ids = Array.isArray(args && args.ids) ? args.ids.map(String).filter(Boolean) : []
|
|
1668
|
+
if (!ids.length) return { ok: false, error: '没有选中要删除的条目' }
|
|
1669
|
+
try {
|
|
1670
|
+
const r = await removeEntries(ids)
|
|
1671
|
+
log('warn', '人工删除记忆 ' + r.deleted + ' 条:' + r.gone.slice(0, 5).join(', ') + (r.gone.length > 5 ? ' …' : '') + (r.indexError ? '' : '(索引同步删除 ' + r.indexDeleted + ' 行)'))
|
|
1672
|
+
store.meta.lastOp = { at: nowMs(), op: 'remove', text: r.deleted + ' 条' }
|
|
1673
|
+
await refreshCount()
|
|
1674
|
+
await persist()
|
|
1675
|
+
return { ok: true, snapshot: snapshot(), deleted: r.deleted, indexDeleted: r.indexDeleted, indexError: r.indexError }
|
|
1676
|
+
} catch (e) {
|
|
1677
|
+
logFailure('删除失败', e, 'removeKnowledge')
|
|
1678
|
+
await persist()
|
|
1679
|
+
return { ok: false, error: msgOf(e), snapshot: snapshot() }
|
|
1680
|
+
}
|
|
1681
|
+
})
|
|
1682
|
+
|
|
1683
|
+
harness.handle('dropCollection', async function () {
|
|
1684
|
+
await ensureLoaded()
|
|
1685
|
+
try {
|
|
1686
|
+
await milvusDrop()
|
|
1687
|
+
log('warn', '已删除 collection ' + collectionName() + '(下次写入会自动重建)')
|
|
1688
|
+
await persist()
|
|
1689
|
+
return { ok: true, snapshot: snapshot() }
|
|
1690
|
+
} catch (e) {
|
|
1691
|
+
logFailure('删除 collection 失败', e, 'dropCollection')
|
|
1692
|
+
await persist()
|
|
1693
|
+
return { ok: false, error: msgOf(e), snapshot: snapshot() }
|
|
1694
|
+
}
|
|
1695
|
+
})
|
|
1696
|
+
|
|
1697
|
+
harness.handle('logClear', async function () {
|
|
1698
|
+
store.log = []
|
|
1699
|
+
await persist()
|
|
1700
|
+
return { ok: true, snapshot: snapshot() }
|
|
1701
|
+
})
|
|
1702
|
+
|
|
1703
|
+
ensureLoaded()
|
|
1704
|
+
.then(function () {
|
|
1705
|
+
console.log('[rtmemory] 本地记忆库 ' + (store.meta.localCount || 0) + ' 条,索引 ' + (store.meta.indexed || 0)
|
|
1706
|
+
+ ' 条,落盘 ' + (store.meta.persistence || '?') + ':' + (store.meta.storePath || '(未解析)'))
|
|
1707
|
+
})
|
|
1708
|
+
.catch(function (e) { console.error('[rtmemory] load failed:', msgOf(e)) })
|
|
1709
|
+
.then(function () { loaded = true })
|
|
1710
|
+
|
|
1711
|
+
console.log('[rtmemory] redteam-memory host half ready; tools = 3, seed =', SEED_KNOWLEDGE.length, ', capture =', capturePhrases().length, '触发词')
|