local-knowledge-graph 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HELP.md +1 -0
- package/lib/agent.js +4 -4
- package/lib/ask.js +59 -1
- package/lib/db.js +204 -9
- package/lib/embeddings.js +5 -1
- package/lib/ingest.js +355 -0
- package/lib/rdf.js +5 -0
- package/lib/similar.js +81 -0
- package/lib/validator.js +26 -2
- package/mcp/server.js +18 -0
- package/package.json +3 -2
- package/public/app.js +352 -28
- package/public/index.html +30 -1
- package/public/style.css +20 -0
- package/server.js +95 -1
package/lib/ingest.js
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// 文档批量入图:上传 md/txt/pdf → 分片 → LLM抽取候选三元组 → 人工审核 → 批量入库
|
|
4
|
+
// 任务持久化于 data/ingest_tasks/<id>.json,服务重启后 review 态可继续。
|
|
5
|
+
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const crypto = require('crypto');
|
|
9
|
+
const { DATA_DIR } = require('./paths');
|
|
10
|
+
const db = require('./db');
|
|
11
|
+
const agent = require('./agent');
|
|
12
|
+
|
|
13
|
+
const TASK_DIR = path.join(DATA_DIR, 'ingest_tasks');
|
|
14
|
+
const MAX_SIZE = 10 * 1024 * 1024;
|
|
15
|
+
const CHUNK_SIZE = 3000;
|
|
16
|
+
const ACCEPT = ['.md', '.markdown', '.txt', '.pdf'];
|
|
17
|
+
const CONFIDENCE = ['确证', '推测', '存疑'];
|
|
18
|
+
|
|
19
|
+
const tasks = new Map(); // id -> task
|
|
20
|
+
|
|
21
|
+
function ensureDir() { fs.mkdirSync(TASK_DIR, { recursive: true }); }
|
|
22
|
+
|
|
23
|
+
function persist(task) {
|
|
24
|
+
ensureDir();
|
|
25
|
+
fs.writeFileSync(path.join(TASK_DIR, task.id + '.json'), JSON.stringify(task, null, 2));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function loadPersisted() {
|
|
29
|
+
ensureDir();
|
|
30
|
+
for (const f of fs.readdirSync(TASK_DIR)) {
|
|
31
|
+
if (!f.endsWith('.json')) continue;
|
|
32
|
+
try {
|
|
33
|
+
const t = JSON.parse(fs.readFileSync(path.join(TASK_DIR, f), 'utf8'));
|
|
34
|
+
if (t.status === 'extracting' || t.status === 'parsing') t.status = 'interrupted';
|
|
35
|
+
tasks.set(t.id, t);
|
|
36
|
+
} catch (_) { /* 损坏任务文件跳过 */ }
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function listTasks() {
|
|
41
|
+
return [...tasks.values()]
|
|
42
|
+
.sort((a, b) => (b.created_at || '').localeCompare(a.created_at || ''))
|
|
43
|
+
.map((t) => ({ id: t.id, filename: t.filename, status: t.status, chunks_total: t.chunks_total, failed_chunks: (t.failed_chunks || []).length, entity_count: t.candidates.entities.length, relation_count: t.candidates.relations.length, created_at: t.created_at, error: t.error || null }));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function getTask(id) { return tasks.get(id) || null; }
|
|
47
|
+
|
|
48
|
+
// ---------- 解析与分片 ----------
|
|
49
|
+
async function extractText(filename, buffer) {
|
|
50
|
+
const ext = path.extname(filename).toLowerCase();
|
|
51
|
+
if (ext === '.pdf') {
|
|
52
|
+
const pdfParse = require('pdf-parse');
|
|
53
|
+
const r = await pdfParse(buffer);
|
|
54
|
+
return r.text || '';
|
|
55
|
+
}
|
|
56
|
+
return buffer.toString('utf8');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function chunkText(text) {
|
|
60
|
+
const paras = text.split(/\n{2,}/).map((s) => s.trim()).filter(Boolean);
|
|
61
|
+
const chunks = [];
|
|
62
|
+
let cur = '';
|
|
63
|
+
for (const p of paras) {
|
|
64
|
+
if (p.length > CHUNK_SIZE) {
|
|
65
|
+
if (cur) { chunks.push(cur); cur = ''; }
|
|
66
|
+
for (let i = 0; i < p.length; i += CHUNK_SIZE) chunks.push(p.slice(i, i + CHUNK_SIZE));
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if ((cur + '\n\n' + p).length > CHUNK_SIZE) { chunks.push(cur); cur = p; }
|
|
70
|
+
else cur = cur ? cur + '\n\n' + p : p;
|
|
71
|
+
}
|
|
72
|
+
if (cur) chunks.push(cur);
|
|
73
|
+
return chunks;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// ---------- LLM 抽取 ----------
|
|
77
|
+
function extractPrompt(chunk) {
|
|
78
|
+
return `你是三元组抽取器。禁止调用任何工具、禁止写入图谱、禁止执行操作,只阅读文本并直接输出JSON。
|
|
79
|
+
严格输出JSON(禁止多余文字/代码块标记/解释),结构:
|
|
80
|
+
{"entities":[{"name":"实体名","category":"物理实体|抽象实体|数值实体|时间实体","attributes":{"扁平键":"原子值"},"aliases":["别名"]}],"relations":[{"from":"起点实体名","to":"终点实体名","name":"关系名","category":"空间|互动|归属|时间|属性","confidence":"确证|推测|存疑"}]}
|
|
81
|
+
要求:实体类别从4类中选最贴切的一种;关系类别从5类中选;confidence按文本依据强度标注,无把握用"推测";属性值只能是字符串/数值/布尔;只依据文本内容,禁止编造;没有可抽取内容时输出 {"entities":[],"relations":[]}。
|
|
82
|
+
|
|
83
|
+
文本:
|
|
84
|
+
${chunk}`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function safeParseJson(text) {
|
|
88
|
+
if (!text) return null;
|
|
89
|
+
let t = String(text).trim();
|
|
90
|
+
const m = t.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
91
|
+
if (m) t = m[1].trim();
|
|
92
|
+
const start = t.indexOf('{');
|
|
93
|
+
const end = t.lastIndexOf('}');
|
|
94
|
+
if (start === -1 || end === -1 || end <= start) return null;
|
|
95
|
+
try { return JSON.parse(t.slice(start, end + 1)); } catch (_) {
|
|
96
|
+
try { return JSON.parse(t.slice(start, end + 1).replace(/,\s*([}\]])/g, '$1')); } catch (_) { return null; }
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function normalizeCandidates(raw, filename, chunkIdx) {
|
|
101
|
+
const srcRef = `《${filename}》片段${chunkIdx + 1}`;
|
|
102
|
+
const entities = [];
|
|
103
|
+
const relations = [];
|
|
104
|
+
if (raw && Array.isArray(raw.entities)) {
|
|
105
|
+
for (const e of raw.entities) {
|
|
106
|
+
if (!e || typeof e.name !== 'string' || !e.name.trim()) continue;
|
|
107
|
+
entities.push({
|
|
108
|
+
name: e.name.trim(),
|
|
109
|
+
category: ['物理实体', '抽象实体', '数值实体', '时间实体'].includes(e.category) ? e.category : '抽象实体',
|
|
110
|
+
attributes: (e.attributes && typeof e.attributes === 'object' && !Array.isArray(e.attributes)) ? e.attributes : {},
|
|
111
|
+
aliases: Array.isArray(e.aliases) ? e.aliases.filter((a) => typeof a === 'string' && a.trim()).map((a) => a.trim()).slice(0, 10) : [],
|
|
112
|
+
existing_id: null,
|
|
113
|
+
selected: true,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (raw && Array.isArray(raw.relations)) {
|
|
118
|
+
for (const r of raw.relations) {
|
|
119
|
+
if (!r || typeof r.from !== 'string' || typeof r.to !== 'string' || typeof r.name !== 'string') continue;
|
|
120
|
+
if (!r.from.trim() || !r.to.trim() || !r.name.trim()) continue;
|
|
121
|
+
relations.push({
|
|
122
|
+
from: r.from.trim(),
|
|
123
|
+
to: r.to.trim(),
|
|
124
|
+
name: r.name.trim(),
|
|
125
|
+
category: ['空间', '互动', '归属', '时间', '属性'].includes(r.category) ? r.category : '属性',
|
|
126
|
+
confidence: CONFIDENCE.includes(r.confidence) ? r.confidence : '推测',
|
|
127
|
+
source_ref: srcRef,
|
|
128
|
+
existing_relation: null,
|
|
129
|
+
selected: true,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return { entities, relations };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// 实体自动匹配:主名→别名(同时检测候选之间重复)
|
|
137
|
+
// 索引值区分两种命中:{entity_id} 指向现有实体;{cand} 指向候选下标
|
|
138
|
+
function matchEntities(task) {
|
|
139
|
+
const nameIndex = new Map();
|
|
140
|
+
for (const e of db.listEntities()) {
|
|
141
|
+
nameIndex.set(e.name, { entity_id: e.id });
|
|
142
|
+
for (const a of db.listAliases(e.id)) if (!nameIndex.has(a.alias)) nameIndex.set(a.alias, { entity_id: e.id });
|
|
143
|
+
}
|
|
144
|
+
task.candidates.entities.forEach((c, i) => {
|
|
145
|
+
if (nameIndex.has(c.name)) {
|
|
146
|
+
const v = nameIndex.get(c.name);
|
|
147
|
+
if (v.entity_id !== undefined) { c.existing_id = v.entity_id; return; }
|
|
148
|
+
c.existing_id = v.cand;
|
|
149
|
+
c.dupe_of_candidate = true;
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
nameIndex.set(c.name, { cand: i });
|
|
153
|
+
for (const a of c.aliases) if (!nameIndex.has(a)) nameIndex.set(a, { cand: i });
|
|
154
|
+
});
|
|
155
|
+
const resolve = (name) => {
|
|
156
|
+
const v = nameIndex.get(name);
|
|
157
|
+
return v ? { ...v } : null;
|
|
158
|
+
};
|
|
159
|
+
for (const r of task.candidates.relations) {
|
|
160
|
+
r.from_ref = resolve(r.from);
|
|
161
|
+
r.to_ref = resolve(r.to);
|
|
162
|
+
r.unresolved = !r.from_ref || !r.to_ref;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// ---------- 任务流程 ----------
|
|
167
|
+
async function createTask(filename, buffer, opts = {}) {
|
|
168
|
+
const { runPlain, autoCommit = false } = opts;
|
|
169
|
+
const ext = path.extname(filename || '').toLowerCase();
|
|
170
|
+
if (!ACCEPT.includes(ext)) { const e = new Error(`仅支持 ${ACCEPT.join(' / ')} 格式`); e.status = 400; throw e; }
|
|
171
|
+
if (!buffer || buffer.length > MAX_SIZE) { const e = new Error('文件超过10MB上限'); e.status = 413; throw e; }
|
|
172
|
+
|
|
173
|
+
const dupCount = [...tasks.values()].filter((t) => t.filename === filename).length;
|
|
174
|
+
const task = {
|
|
175
|
+
id: crypto.randomBytes(6).toString('hex'),
|
|
176
|
+
filename,
|
|
177
|
+
display_name: dupCount ? `${filename}(第${dupCount + 1}次导入)` : filename,
|
|
178
|
+
status: 'parsing',
|
|
179
|
+
chunks_total: 0,
|
|
180
|
+
failed_chunks: [],
|
|
181
|
+
candidates: { entities: [], relations: [] },
|
|
182
|
+
auto_commit: !!autoCommit,
|
|
183
|
+
created_at: new Date().toISOString(),
|
|
184
|
+
committed_at: null,
|
|
185
|
+
error: null,
|
|
186
|
+
};
|
|
187
|
+
tasks.set(task.id, task);
|
|
188
|
+
storeUpload(task.id, filename, buffer);
|
|
189
|
+
persist(task);
|
|
190
|
+
|
|
191
|
+
// 异步执行,接口立即返回任务id;抽取走只读模式,杜绝Agent写库
|
|
192
|
+
run(task, buffer, runPlain || ((p, t) => agent.runPlain(p, t || 180000, { KG_MCP_READONLY: '1' })))
|
|
193
|
+
.catch((e) => {
|
|
194
|
+
task.status = 'failed';
|
|
195
|
+
task.error = e.message;
|
|
196
|
+
persist(task);
|
|
197
|
+
})
|
|
198
|
+
.finally(() => { if (typeof opts.onSettled === 'function') { try { opts.onSettled(); } catch (_) {} } });
|
|
199
|
+
return { id: task.id, status: task.status };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async function run(task, buffer, runPlain) {
|
|
203
|
+
// 1) 解析
|
|
204
|
+
let text;
|
|
205
|
+
try {
|
|
206
|
+
text = await extractText(task.filename, buffer);
|
|
207
|
+
} catch (e) {
|
|
208
|
+
task.status = 'failed';
|
|
209
|
+
task.error = '文档解析失败: ' + e.message;
|
|
210
|
+
persist(task);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
if (!text.trim()) {
|
|
214
|
+
task.status = 'failed';
|
|
215
|
+
task.error = '文档无可提取文本';
|
|
216
|
+
persist(task);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const chunks = chunkText(text);
|
|
220
|
+
task.chunks_total = chunks.length;
|
|
221
|
+
task.status = 'extracting';
|
|
222
|
+
persist(task);
|
|
223
|
+
|
|
224
|
+
// 2) 逐片抽取
|
|
225
|
+
const raws = [];
|
|
226
|
+
task.chunk_errors = task.chunk_errors || [];
|
|
227
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
228
|
+
try {
|
|
229
|
+
const out = await runPlain(extractPrompt(chunks[i]));
|
|
230
|
+
const outText = typeof out === 'string' ? out : (out && out.ok === false ? null : String((out && out.text) || ''));
|
|
231
|
+
if (outText === null) { task.chunk_errors.push({ chunk: i, error: (out && out.error) || '空结果' }); task.failed_chunks.push(i); continue; }
|
|
232
|
+
const parsed = safeParseJson(outText);
|
|
233
|
+
if (!parsed) { task.chunk_errors.push({ chunk: i, error: 'LLM输出无法解析为JSON:' + outText.slice(0, 120) }); task.failed_chunks.push(i); continue; }
|
|
234
|
+
raws.push(normalizeCandidates(parsed, task.filename, i));
|
|
235
|
+
} catch (e) {
|
|
236
|
+
task.chunk_errors.push({ chunk: i, error: e.message });
|
|
237
|
+
task.failed_chunks.push(i);
|
|
238
|
+
if (i === chunks.length - 1 && !raws.length && e.message && /超时|timeout/i.test(e.message)) {
|
|
239
|
+
task.error = 'LLM调用超时: ' + e.message;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
// 3) 合并候选(同名实体去重合并别名)
|
|
244
|
+
for (const r of raws) {
|
|
245
|
+
for (const e of r.entities) {
|
|
246
|
+
const exist = task.candidates.entities.find((x) => x.name === e.name);
|
|
247
|
+
if (exist) { for (const a of e.aliases) if (!exist.aliases.includes(a)) exist.aliases.push(a); for (const [k, v] of Object.entries(e.attributes)) if (!(k in exist.attributes)) exist.attributes[k] = v; }
|
|
248
|
+
else task.candidates.entities.push(e);
|
|
249
|
+
}
|
|
250
|
+
task.candidates.relations.push(...r.relations);
|
|
251
|
+
}
|
|
252
|
+
// 关系去重(from+name+to)
|
|
253
|
+
const seen = new Set();
|
|
254
|
+
task.candidates.relations = task.candidates.relations.filter((r) => {
|
|
255
|
+
const key = `${r.from}|${r.name}|${r.to}`;
|
|
256
|
+
if (seen.has(key)) return false;
|
|
257
|
+
seen.add(key);
|
|
258
|
+
return true;
|
|
259
|
+
});
|
|
260
|
+
matchEntities(task);
|
|
261
|
+
|
|
262
|
+
if (!task.candidates.entities.length && !task.candidates.relations.length) {
|
|
263
|
+
task.status = 'failed';
|
|
264
|
+
task.error = task.failed_chunks.length
|
|
265
|
+
? (task.error || `全部${task.chunks_total}个片段抽取失败` + (task.chunk_errors.length ? `(首个错误: ${task.chunk_errors[0].error})` : ''))
|
|
266
|
+
: '未抽取到候选三元组';
|
|
267
|
+
persist(task);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
task.status = 'review';
|
|
271
|
+
persist(task);
|
|
272
|
+
|
|
273
|
+
// 4) 跳过审核直通
|
|
274
|
+
if (task.auto_commit) commitTask(task.id, null, '系统');
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function commitTask(id, selected, source) {
|
|
278
|
+
const task = tasks.get(id);
|
|
279
|
+
if (!task) { const e = new Error(`任务${id}不存在`); e.status = 404; throw e; }
|
|
280
|
+
if (task.status !== 'review') { const e = new Error(`任务状态为${task.status},无法提交`); e.status = 400; throw e; }
|
|
281
|
+
const ents = task.candidates.entities.filter((c) => c.selected && (!c.dupe_of_candidate));
|
|
282
|
+
const rels = task.candidates.relations.filter((r) => r.selected && !r.unresolved);
|
|
283
|
+
if (selected) {
|
|
284
|
+
const entNames = new Set((selected.entities || []).map((s) => String(s)));
|
|
285
|
+
const relKeys = new Set(selected.relations || []);
|
|
286
|
+
for (const c of ents) c._want = entNames.has(c.name);
|
|
287
|
+
for (const r of rels) r._want = relKeys.has(`${r.from}|${r.name}|${r.to}`);
|
|
288
|
+
} else {
|
|
289
|
+
for (const c of ents) c._want = true;
|
|
290
|
+
for (const r of rels) r._want = true;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const applied = null;
|
|
294
|
+
void applied;
|
|
295
|
+
// 先打保存点再写入
|
|
296
|
+
const git = require('./git');
|
|
297
|
+
git.savepoint(`文档入图: ${task.display_name}`, source === '系统' ? '系统' : '手工');
|
|
298
|
+
|
|
299
|
+
const nameToId = new Map();
|
|
300
|
+
let entAdded = 0, entLinked = 0, relAdded = 0, relSkipped = 0;
|
|
301
|
+
const opsLog = [];
|
|
302
|
+
for (const c of ents) {
|
|
303
|
+
if (!c._want) continue;
|
|
304
|
+
if (c.existing_id !== null && c.existing_id !== undefined && typeof c.existing_id === 'number') {
|
|
305
|
+
nameToId.set(c.name, c.existing_id);
|
|
306
|
+
entLinked += 1;
|
|
307
|
+
for (const a of c.aliases) {
|
|
308
|
+
try { db.addAlias(c.existing_id, a, source === '系统' ? '系统' : '手工'); } catch (_) { /* 别名冲突忽略 */ }
|
|
309
|
+
}
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
const created = db.addEntity({ name: c.name, category: c.category, attributes: c.attributes }, '文档');
|
|
313
|
+
nameToId.set(c.name, created.id);
|
|
314
|
+
entAdded += 1;
|
|
315
|
+
for (const a of c.aliases) {
|
|
316
|
+
try { db.addAlias(created.id, a, source === '系统' ? '系统' : '手工'); } catch (_) {}
|
|
317
|
+
}
|
|
318
|
+
opsLog.push({ type: 'ADD_ENTITY', name: c.name, id: created.id });
|
|
319
|
+
}
|
|
320
|
+
for (const r of rels) {
|
|
321
|
+
if (!r._want) { relSkipped += 1; continue; }
|
|
322
|
+
const fromId = r.from_ref && r.from_ref.entity_id ? r.from_ref.entity_id : nameToId.get(r.from);
|
|
323
|
+
const toId = r.to_ref && r.to_ref.entity_id ? r.to_ref.entity_id : nameToId.get(r.to);
|
|
324
|
+
if (!fromId || !toId || fromId === toId) { relSkipped += 1; continue; }
|
|
325
|
+
try {
|
|
326
|
+
const created = db.addRelation({ source_id: fromId, target_id: toId, name: r.name, category: r.category, confidence: r.confidence, source_ref: r.source_ref }, '文档');
|
|
327
|
+
relAdded += 1;
|
|
328
|
+
opsLog.push({ type: 'ADD_RELATION', id: created.id, name: r.name, from: fromId, to: toId });
|
|
329
|
+
} catch (_) { relSkipped += 1; }
|
|
330
|
+
}
|
|
331
|
+
task.status = 'committed';
|
|
332
|
+
task.committed_at = new Date().toISOString();
|
|
333
|
+
task.result = { entities_added: entAdded, entities_linked: entLinked, relations_added: relAdded, relations_skipped: relSkipped };
|
|
334
|
+
persist(task);
|
|
335
|
+
return { ok: true, ...task.result, savepoint: task.display_name };
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function deleteTask(id) {
|
|
339
|
+
const task = tasks.get(id);
|
|
340
|
+
if (!task) { const e = new Error(`任务${id}不存在`); e.status = 404; throw e; }
|
|
341
|
+
if (task.status === 'extracting' || task.status === 'parsing') { const e = new Error('任务执行中,请稍后再删除'); e.status = 400; throw e; }
|
|
342
|
+
tasks.delete(id);
|
|
343
|
+
try { fs.unlinkSync(path.join(TASK_DIR, id + '.json')); } catch (_) {}
|
|
344
|
+
return { deleted: id };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// 上传文件暂存(供审核期间重试/溯源)
|
|
348
|
+
function storeUpload(id, filename, buffer) {
|
|
349
|
+
ensureDir();
|
|
350
|
+
const dir = path.join(DATA_DIR, 'uploads');
|
|
351
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
352
|
+
fs.writeFileSync(path.join(dir, id + '_' + filename), buffer);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
module.exports = { createTask, listTasks, getTask, commitTask, deleteTask, storeUpload, loadPersisted, chunkText, safeParseJson, normalizeCandidates, matchEntities, extractText, extractPrompt, ACCEPT };
|
package/lib/rdf.js
CHANGED
|
@@ -42,6 +42,9 @@ function exportTurtle(graph) {
|
|
|
42
42
|
for (const [k, v] of Object.entries(attrs)) {
|
|
43
43
|
chunks.push(` kg:attribute [ kg:key ${lit(k)} ; kg:value ${lit(v)} ]`);
|
|
44
44
|
}
|
|
45
|
+
for (const alias of (graph.aliases && graph.aliases[e.id]) || []) {
|
|
46
|
+
chunks.push(` kg:alias ${lit(alias)}`);
|
|
47
|
+
}
|
|
45
48
|
parts.push(chunks.length ? lines.join('\n') + ' ;\n' + chunks.join(' ;\n') + ' .\n' : lines.join('\n') + ' .\n');
|
|
46
49
|
}
|
|
47
50
|
|
|
@@ -53,6 +56,8 @@ function exportTurtle(graph) {
|
|
|
53
56
|
` kg:category "${esc(r.category)}" ;\n` +
|
|
54
57
|
` kg:subject kg:e${r.source_id} ;\n` +
|
|
55
58
|
` kg:object kg:e${r.target_id} ;\n` +
|
|
59
|
+
` kg:confidence "${esc(r.confidence || '确证')}" ;\n` +
|
|
60
|
+
` kg:sourceRef ${lit(r.source_ref || '')} ;\n` +
|
|
56
61
|
` kg:createdAt "${esc(r.created_at)}"^^xsd:dateTime ;\n` +
|
|
57
62
|
` kg:source "${esc(r.source)}" .\n`
|
|
58
63
|
);
|
package/lib/similar.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// 相似实体:优先嵌入向量余弦Top-K,向量不可用时降级为共邻居Jaccard
|
|
4
|
+
|
|
5
|
+
const db = require('./db');
|
|
6
|
+
const embeddings = require('./embeddings');
|
|
7
|
+
const vectors = require('./vectors');
|
|
8
|
+
|
|
9
|
+
function coNeighborScores(targetId, entities, relations) {
|
|
10
|
+
const neighbors = new Map();
|
|
11
|
+
for (const r of relations) {
|
|
12
|
+
if (r.source_id === targetId) neighbors.set(r.target_id, 1);
|
|
13
|
+
if (r.target_id === targetId) neighbors.set(r.source_id, 1);
|
|
14
|
+
}
|
|
15
|
+
if (!neighbors.size) return [];
|
|
16
|
+
const incident = new Map();
|
|
17
|
+
for (const r of relations) {
|
|
18
|
+
for (const [a, b] of [[r.source_id, r.target_id], [r.target_id, r.source_id]]) {
|
|
19
|
+
if (!incident.has(a)) incident.set(a, new Set());
|
|
20
|
+
incident.get(a).add(b);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const own = incident.get(targetId) || new Set();
|
|
24
|
+
const scores = [];
|
|
25
|
+
for (const e of entities) {
|
|
26
|
+
if (e.id === targetId) continue;
|
|
27
|
+
const other = incident.get(e.id);
|
|
28
|
+
if (!other || !other.size) continue;
|
|
29
|
+
let inter = 0;
|
|
30
|
+
for (const n of other) if (own.has(n)) inter += 1;
|
|
31
|
+
if (inter === 0 && !neighbors.has(e.id)) continue;
|
|
32
|
+
const union = new Set([...own, ...other]).size;
|
|
33
|
+
scores.push({ id: e.id, score: inter / union });
|
|
34
|
+
}
|
|
35
|
+
return scores;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function similarEntities(id, k = 8) {
|
|
39
|
+
const target = db.getEntity(id);
|
|
40
|
+
if (!target) { const e = new Error(`实体id=${id} 不存在`); e.status = 404; throw e; }
|
|
41
|
+
const cap = Math.min(20, Math.max(1, Number.isInteger(k) ? k : 8));
|
|
42
|
+
const graph = db.getGraph();
|
|
43
|
+
const byId = new Map(graph.entities.map((e) => [e.id, e]));
|
|
44
|
+
|
|
45
|
+
// 语义路径:目标向量缺失时即时补算
|
|
46
|
+
let semantic = [];
|
|
47
|
+
const s = embeddings.loadSettings();
|
|
48
|
+
if (s.api_key && vectors.count() > 0) {
|
|
49
|
+
try {
|
|
50
|
+
let trow = vectors.get(id);
|
|
51
|
+
if (!trow) {
|
|
52
|
+
const [vec] = await embeddings.embed([embeddings.entityText(target)], s);
|
|
53
|
+
vectors.upsert(id, Float32Array.from(vec));
|
|
54
|
+
trow = { entity_id: id, vector: Float32Array.from(vec) };
|
|
55
|
+
}
|
|
56
|
+
const tvec = Float32Array.from(trow.vector);
|
|
57
|
+
semantic = vectors.all()
|
|
58
|
+
.filter((row) => row.entity_id !== id)
|
|
59
|
+
.map((row) => ({ id: row.entity_id, score: embeddings.cosine(tvec, Float32Array.from(row.vector)) }))
|
|
60
|
+
.filter((x) => x.score > 0.1);
|
|
61
|
+
} catch (_) { semantic = []; }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (semantic.length) {
|
|
65
|
+
semantic.sort((a, b) => b.score - a.score);
|
|
66
|
+
return {
|
|
67
|
+
mode: 'semantic',
|
|
68
|
+
results: semantic.slice(0, cap).map((x) => ({ entity: byId.get(x.id), score: Number(x.score.toFixed(4)) })).filter((x) => x.entity),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// 结构降级:共邻居Jaccard
|
|
73
|
+
const struct = coNeighborScores(id, graph.entities, graph.relations)
|
|
74
|
+
.sort((a, b) => b.score - a.score)
|
|
75
|
+
.slice(0, cap)
|
|
76
|
+
.map((x) => ({ entity: byId.get(x.id), score: Number(x.score.toFixed(4)) }))
|
|
77
|
+
.filter((x) => x.entity);
|
|
78
|
+
return { mode: 'structure', results: struct };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { similarEntities };
|
package/lib/validator.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
const ENTITY_CATEGORIES = ['物理实体', '抽象实体', '数值实体', '时间实体'];
|
|
6
6
|
const RELATION_CATEGORIES = ['空间', '互动', '归属', '时间', '属性'];
|
|
7
|
-
const SOURCES = ['手工', 'OpenCode', '系统'];
|
|
7
|
+
const SOURCES = ['手工', 'OpenCode', '系统', '文档'];
|
|
8
|
+
const CONFIDENCE_LEVELS = ['确证', '推测', '存疑'];
|
|
8
9
|
const RDF_TYPE_MAP = {
|
|
9
10
|
'物理实体': 'PhysicalEntity',
|
|
10
11
|
'抽象实体': 'AbstractEntity',
|
|
@@ -106,8 +107,17 @@ function validateRelationInput(input, db) {
|
|
|
106
107
|
errors.push(`关系大类必须为以下5类之一: ${RELATION_CATEGORIES.join(' / ')}`);
|
|
107
108
|
}
|
|
108
109
|
|
|
110
|
+
const confidence = input.confidence === undefined || input.confidence === null || input.confidence === '' ? '确证' : input.confidence;
|
|
111
|
+
if (!CONFIDENCE_LEVELS.includes(confidence)) {
|
|
112
|
+
errors.push(`置信度必须为以下3档之一: ${CONFIDENCE_LEVELS.join(' / ')}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const source_ref = input.source_ref === undefined || input.source_ref === null ? '' : input.source_ref;
|
|
116
|
+
if (typeof source_ref !== 'string') errors.push('来源引用(source_ref)必须为字符串');
|
|
117
|
+
else if (source_ref.length > 500) errors.push('来源引用长度不得超过500字符');
|
|
118
|
+
|
|
109
119
|
if (errors.length) return { ok: false, errors, value: null };
|
|
110
|
-
return { ok: true, errors: [], value: { source_id: sid, target_id: tid, name: name.trim(), category } };
|
|
120
|
+
return { ok: true, errors: [], value: { source_id: sid, target_id: tid, name: name.trim(), category, confidence, source_ref } };
|
|
111
121
|
}
|
|
112
122
|
|
|
113
123
|
function validateRelationPatch(patch, existing, db) {
|
|
@@ -117,6 +127,8 @@ function validateRelationPatch(patch, existing, db) {
|
|
|
117
127
|
target_id: existing.target_id,
|
|
118
128
|
name: existing.name,
|
|
119
129
|
category: existing.category,
|
|
130
|
+
confidence: existing.confidence || '确证',
|
|
131
|
+
source_ref: existing.source_ref || '',
|
|
120
132
|
};
|
|
121
133
|
if (patch.source_id !== undefined) merged.source_id = patch.source_id;
|
|
122
134
|
if (patch.target_id !== undefined) merged.target_id = patch.target_id;
|
|
@@ -128,6 +140,17 @@ function validateRelationPatch(patch, existing, db) {
|
|
|
128
140
|
if (!RELATION_CATEGORIES.includes(patch.category)) errors.push(`关系大类必须为: ${RELATION_CATEGORIES.join(' / ')}`);
|
|
129
141
|
else merged.category = patch.category;
|
|
130
142
|
}
|
|
143
|
+
if (patch.confidence !== undefined) {
|
|
144
|
+
if (patch.confidence === null || patch.confidence === '') merged.confidence = '确证';
|
|
145
|
+
else if (!CONFIDENCE_LEVELS.includes(patch.confidence)) errors.push(`置信度必须为: ${CONFIDENCE_LEVELS.join(' / ')}`);
|
|
146
|
+
else merged.confidence = patch.confidence;
|
|
147
|
+
}
|
|
148
|
+
if (patch.source_ref !== undefined) {
|
|
149
|
+
if (patch.source_ref === null) merged.source_ref = '';
|
|
150
|
+
else if (typeof patch.source_ref !== 'string') errors.push('来源引用(source_ref)必须为字符串');
|
|
151
|
+
else if (patch.source_ref.length > 500) errors.push('来源引用长度不得超过500字符');
|
|
152
|
+
else merged.source_ref = patch.source_ref;
|
|
153
|
+
}
|
|
131
154
|
const check = validateRelationInput(merged, db);
|
|
132
155
|
if (errors.length || !check.ok) return { ok: false, errors: [...errors, ...check.errors], value: null };
|
|
133
156
|
return check;
|
|
@@ -137,6 +160,7 @@ module.exports = {
|
|
|
137
160
|
ENTITY_CATEGORIES,
|
|
138
161
|
RELATION_CATEGORIES,
|
|
139
162
|
SOURCES,
|
|
163
|
+
CONFIDENCE_LEVELS,
|
|
140
164
|
RDF_TYPE_MAP,
|
|
141
165
|
RDF_RELATION_TYPE_MAP,
|
|
142
166
|
validateEntityInput,
|
package/mcp/server.js
CHANGED
|
@@ -98,6 +98,19 @@ const TOOLS = [
|
|
|
98
98
|
properties: { center: { type: 'string', description: '可选,实体id或名称' } },
|
|
99
99
|
},
|
|
100
100
|
},
|
|
101
|
+
{
|
|
102
|
+
name: 'kg_path',
|
|
103
|
+
description: '两实体关系路径枚举:返回最多5条按跳数升序的路径(每跳含关系名/类别/置信度),用于验证两个对象如何间接关联',
|
|
104
|
+
inputSchema: {
|
|
105
|
+
type: 'object',
|
|
106
|
+
properties: {
|
|
107
|
+
from: { type: 'string', description: '起点实体id或名称' },
|
|
108
|
+
to: { type: 'string', description: '终点实体id或名称' },
|
|
109
|
+
max: { type: 'number', description: '可选,最大跳数2-6,默认4' },
|
|
110
|
+
},
|
|
111
|
+
required: ['from', 'to'],
|
|
112
|
+
},
|
|
113
|
+
},
|
|
101
114
|
{
|
|
102
115
|
name: 'kg_digest',
|
|
103
116
|
description: '图谱目录:实体大类、关系大类、高频关系名Top20(含数量)、样例实体、规模。回答关系类问题前先取此目录,可显著提升Cypher查询的准确性',
|
|
@@ -200,6 +213,11 @@ const HANDLERS = {
|
|
|
200
213
|
const { digest, stats } = require('./lib/ask').buildDigest();
|
|
201
214
|
return { digest, ...stats };
|
|
202
215
|
},
|
|
216
|
+
kg_path: async (args) => {
|
|
217
|
+
const from = resolveCenter(String(args.from || ''));
|
|
218
|
+
const to = resolveCenter(String(args.to || ''));
|
|
219
|
+
return db.findPaths(from.id, to.id, { maxHops: Number.isInteger(args.max) ? args.max : 4 });
|
|
220
|
+
},
|
|
203
221
|
kg_export_rdf: async () => ({ turtle: rdf.exportTurtle(db.getGraph()) }),
|
|
204
222
|
kg_apply_ops: async (args) => {
|
|
205
223
|
if (READONLY) throw new Error('MCP运行于只读模式(KG_MCP_READONLY=1),写入被拒绝');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "local-knowledge-graph",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "本地知识图谱整合器:3D可视化 + AI对话建图 + 智能检索 + Git回溯 + RDF合规,全流程本地运行",
|
|
5
5
|
"main": "server.js",
|
|
6
6
|
"bin": {
|
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
"test": "node --test \"test/*.test.js\""
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"express": "^4.22.3"
|
|
28
|
+
"express": "^4.22.3",
|
|
29
|
+
"pdf-parse": "^2.4.5"
|
|
29
30
|
},
|
|
30
31
|
"keywords": [
|
|
31
32
|
"knowledge-graph",
|