dsh-knowledge-library 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/LICENSE +21 -0
- package/README.md +221 -2
- package/cordis.patch.yml +3 -0
- package/dist/client.d.ts +3 -0
- package/dist/client.js +721 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +105 -0
- package/dist/core/fetch.d.ts +10 -0
- package/dist/core/fetch.js +105 -0
- package/dist/core/jobs.d.ts +30 -0
- package/dist/core/jobs.js +96 -0
- package/dist/core/library.d.ts +61 -0
- package/dist/core/library.js +386 -0
- package/dist/http.d.ts +6 -0
- package/dist/http.js +157 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +35 -0
- package/dist/public/app.css +182 -0
- package/dist/public/app.js +56 -0
- package/dist/public/index.html +1 -0
- package/dist/standalone.d.ts +1 -0
- package/dist/standalone.js +38 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +62 -0
- package/dist/types.d.ts +78 -0
- package/dist/types.js +1 -0
- package/dist/ui/App.d.ts +2 -0
- package/dist/ui/App.js +222 -0
- package/dist/ui/icons.d.ts +8 -0
- package/dist/ui/icons.js +25 -0
- package/dist/ui/main.d.ts +1 -0
- package/dist/ui/main.js +4 -0
- package/package.json +94 -4
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, readdirSync, statSync, lstatSync, unlinkSync, openSync, closeSync } from 'node:fs';
|
|
4
|
+
import { resolve, join, dirname, basename, extname, relative, isAbsolute } from 'node:path';
|
|
5
|
+
import { extractArticle, formatTime, parseSubtitles, splitSegments } from './extract.js';
|
|
6
|
+
import { validateUrl } from './fetch.js';
|
|
7
|
+
export const MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
8
|
+
export const MAX_TEXT_CHARS = 2_000_000;
|
|
9
|
+
const hash = (data) => createHash('sha256').update(data).digest('hex');
|
|
10
|
+
const now = () => new Date().toISOString();
|
|
11
|
+
const cleanString = (value, max = 200) => typeof value === 'string' ? value.trim().slice(0, max) : '';
|
|
12
|
+
export function categoryName(value) {
|
|
13
|
+
const parts = cleanString(value, 180).split(/[\\/]/).filter(Boolean);
|
|
14
|
+
if (parts.length > 5 || parts.some(p => p === '.' || p === '..' || /[<>:"|?*\x00-\x1f]/.test(p) || /[. ]$/.test(p) || /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(?:\.|$)/i.test(p))) {
|
|
15
|
+
throw new Error('分类最多 5 层,不能包含路径跳转、保留名称或特殊文件名字符');
|
|
16
|
+
}
|
|
17
|
+
return parts.join('/') || '收件箱';
|
|
18
|
+
}
|
|
19
|
+
export function atomicJson(path, value) {
|
|
20
|
+
const tmp = `${path}.${randomUUID()}.tmp`;
|
|
21
|
+
writeFileSync(tmp, JSON.stringify(value, null, 2), { encoding: 'utf8', flag: 'wx' });
|
|
22
|
+
renameSync(tmp, path);
|
|
23
|
+
}
|
|
24
|
+
export class Library {
|
|
25
|
+
root;
|
|
26
|
+
db;
|
|
27
|
+
warnings = [];
|
|
28
|
+
lockPath;
|
|
29
|
+
closed = false;
|
|
30
|
+
constructor(root) {
|
|
31
|
+
if (!isAbsolute(root))
|
|
32
|
+
throw new Error('资料库目录必须是绝对路径');
|
|
33
|
+
this.root = resolve(root);
|
|
34
|
+
mkdirSync(join(this.root, '.system'), { recursive: true });
|
|
35
|
+
mkdirSync(join(this.root, 'materials'), { recursive: true });
|
|
36
|
+
this.lockPath = join(this.root, '.system', 'writer.lock');
|
|
37
|
+
if (existsSync(this.lockPath)) {
|
|
38
|
+
const previous = Number(readFileSync(this.lockPath, 'utf8'));
|
|
39
|
+
if (!Number.isInteger(previous) || previous <= 0)
|
|
40
|
+
throw new Error('资料库锁文件无效,请确认没有其他实例后移除 .system/writer.lock');
|
|
41
|
+
let alive = false;
|
|
42
|
+
try {
|
|
43
|
+
process.kill(previous, 0);
|
|
44
|
+
alive = true;
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code !== 'ESRCH')
|
|
48
|
+
alive = true;
|
|
49
|
+
}
|
|
50
|
+
if (alive)
|
|
51
|
+
throw new Error('这个资料库已被另一实例打开,请关闭独立预览或另一个 Harness 实例');
|
|
52
|
+
unlinkSync(this.lockPath);
|
|
53
|
+
}
|
|
54
|
+
const fd = openSync(this.lockPath, 'wx');
|
|
55
|
+
writeFileSync(fd, String(process.pid));
|
|
56
|
+
closeSync(fd);
|
|
57
|
+
try {
|
|
58
|
+
this.db = new DatabaseSync(join(this.root, '.system', 'index.sqlite'));
|
|
59
|
+
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
|
|
60
|
+
CREATE TABLE IF NOT EXISTS materials (id TEXT PRIMARY KEY, path TEXT NOT NULL, meta TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, updated_at TEXT NOT NULL);
|
|
61
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS material_fts USING fts5(id UNINDEXED, title, content, tokenize='trigram');`);
|
|
62
|
+
this.rebuild();
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
try {
|
|
66
|
+
this.db.close();
|
|
67
|
+
}
|
|
68
|
+
catch { }
|
|
69
|
+
unlinkSync(this.lockPath);
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
close() {
|
|
74
|
+
if (this.closed)
|
|
75
|
+
return;
|
|
76
|
+
this.closed = true;
|
|
77
|
+
this.db.close();
|
|
78
|
+
if (existsSync(this.lockPath) && readFileSync(this.lockPath, 'utf8') === String(process.pid))
|
|
79
|
+
unlinkSync(this.lockPath);
|
|
80
|
+
}
|
|
81
|
+
assertContained(path) {
|
|
82
|
+
const rel = relative(this.root, resolve(path));
|
|
83
|
+
if (rel.startsWith('..') || isAbsolute(rel))
|
|
84
|
+
throw new Error('路径超出资料库');
|
|
85
|
+
let current = this.root;
|
|
86
|
+
for (const part of rel.split(/[\\/]/).filter(Boolean)) {
|
|
87
|
+
current = join(current, part);
|
|
88
|
+
if (existsSync(current) && lstatSync(current).isSymbolicLink())
|
|
89
|
+
throw new Error('资料库不支持符号链接目录');
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
index(meta, path, content) {
|
|
93
|
+
this.db.prepare('INSERT OR REPLACE INTO materials VALUES (?,?,?,?,?,?)').run(meta.id, path, JSON.stringify(meta), content, meta.contentHash, meta.updatedAt);
|
|
94
|
+
this.db.prepare('DELETE FROM material_fts WHERE id=?').run(meta.id);
|
|
95
|
+
this.db.prepare('INSERT INTO material_fts (id,title,content) VALUES (?,?,?)').run(meta.id, meta.title, content);
|
|
96
|
+
}
|
|
97
|
+
rebuild() {
|
|
98
|
+
this.warnings.length = 0;
|
|
99
|
+
const entries = [];
|
|
100
|
+
const visit = (directory, depth) => {
|
|
101
|
+
if (depth > 7)
|
|
102
|
+
return;
|
|
103
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
104
|
+
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.isSymbolicLink())
|
|
105
|
+
continue;
|
|
106
|
+
const path = join(directory, entry.name);
|
|
107
|
+
if (!existsSync(join(path, 'metadata.json'))) {
|
|
108
|
+
visit(path, depth + 1);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
this.assertContained(path);
|
|
113
|
+
const meta = JSON.parse(readFileSync(join(path, 'metadata.json'), 'utf8'));
|
|
114
|
+
if (meta.schemaVersion !== 1 || !/^r_[a-f0-9-]+$/.test(meta.id) || meta.id !== entry.name || !Array.isArray(meta.tags) || !Array.isArray(meta.citations))
|
|
115
|
+
throw new Error('元数据格式不正确');
|
|
116
|
+
if (!['text', 'web', 'file', 'video', 'transcript', 'note'].includes(meta.kind))
|
|
117
|
+
throw new Error('未知资料类型');
|
|
118
|
+
const content = readFileSync(join(path, 'content.md'), 'utf8');
|
|
119
|
+
const segmentsRaw = readFileSync(join(path, 'segments.json'), 'utf8');
|
|
120
|
+
const segments = JSON.parse(segmentsRaw);
|
|
121
|
+
if (!Array.isArray(segments))
|
|
122
|
+
throw new Error('缺少可引用段落');
|
|
123
|
+
if ((meta.textChecksum && hash(content) !== meta.textChecksum) || (meta.segmentsChecksum && hash(segmentsRaw) !== meta.segmentsChecksum)) {
|
|
124
|
+
throw new Error('正文或引用段落被外部修改,请恢复原文件或将修改后的内容重新导入');
|
|
125
|
+
}
|
|
126
|
+
// The physical category remains recoverable after an interrupted move.
|
|
127
|
+
meta.category = relative(join(this.root, 'materials'), dirname(path)).split('\\').join('/');
|
|
128
|
+
entries.push({ meta, path, content });
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
this.warnings.push(`${entry.name}: ${error.message}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
visit(join(this.root, 'materials'), 0);
|
|
136
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
137
|
+
try {
|
|
138
|
+
this.db.exec('DELETE FROM materials; DELETE FROM material_fts');
|
|
139
|
+
for (const entry of entries)
|
|
140
|
+
this.index(entry.meta, entry.path, entry.content);
|
|
141
|
+
this.db.exec('COMMIT');
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
this.db.exec('ROLLBACK');
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
return { indexed: entries.length, warnings: [...this.warnings] };
|
|
148
|
+
}
|
|
149
|
+
row(id) {
|
|
150
|
+
const row = this.db.prepare('SELECT * FROM materials WHERE id=?').get(id);
|
|
151
|
+
if (!row)
|
|
152
|
+
throw new Error('资料不存在');
|
|
153
|
+
this.assertContained(row.path);
|
|
154
|
+
return row;
|
|
155
|
+
}
|
|
156
|
+
get(id) {
|
|
157
|
+
const row = this.row(id);
|
|
158
|
+
return { ...JSON.parse(row.meta), content: row.content, segments: JSON.parse(readFileSync(join(row.path, 'segments.json'), 'utf8')), path: row.path };
|
|
159
|
+
}
|
|
160
|
+
original(id) {
|
|
161
|
+
const item = this.get(id);
|
|
162
|
+
if (!item.originalFile || basename(item.originalFile) !== item.originalFile)
|
|
163
|
+
throw new Error('该资料没有原始附件');
|
|
164
|
+
const path = join(item.path, item.originalFile);
|
|
165
|
+
this.assertContained(path);
|
|
166
|
+
return { bytes: readFileSync(path), name: item.originalName || item.originalFile };
|
|
167
|
+
}
|
|
168
|
+
save(content, fields, extra, original, suppliedSegments) {
|
|
169
|
+
if (content.length > MAX_TEXT_CHARS)
|
|
170
|
+
throw new Error('正文超过 200 万字符,请拆分后导入');
|
|
171
|
+
const digest = hash(`${extra.kind}\0${original ? hash(original.bytes) : content}\0${fields.sourceUrl ?? ''}\0${JSON.stringify(extra.citations ?? [])}`);
|
|
172
|
+
const duplicate = this.db.prepare('SELECT id FROM materials WHERE content_hash=? LIMIT 1').get(digest);
|
|
173
|
+
if (duplicate)
|
|
174
|
+
return { material: this.get(duplicate.id), duplicate: true };
|
|
175
|
+
const category = categoryName(fields.category);
|
|
176
|
+
const id = `r_${randomUUID()}`;
|
|
177
|
+
const timestamp = now();
|
|
178
|
+
const meta = {
|
|
179
|
+
schemaVersion: 1, id, title: cleanString(fields.title, 300) || content.split('\n').find(line => line.trim())?.replace(/^#+\s*/, '').slice(0, 100) || '未命名资料',
|
|
180
|
+
status: 'ready', category, tags: this.tags(fields.tags),
|
|
181
|
+
collection: cleanString(fields.collection), sourceNote: cleanString(fields.sourceNote, 2000),
|
|
182
|
+
createdAt: timestamp, updatedAt: timestamp, contentHash: digest, revision: 1, citations: [],
|
|
183
|
+
extraction: '原始文本', ...extra,
|
|
184
|
+
};
|
|
185
|
+
if (fields.sourceUrl)
|
|
186
|
+
meta.sourceUrl = validateUrl(fields.sourceUrl).href;
|
|
187
|
+
const path = join(this.root, 'materials', ...category.split('/'), id);
|
|
188
|
+
this.assertContained(path);
|
|
189
|
+
const stage = join(dirname(path), `.pending-${id}`);
|
|
190
|
+
mkdirSync(stage, { recursive: true });
|
|
191
|
+
if (original) {
|
|
192
|
+
if (original.bytes.length > MAX_FILE_BYTES)
|
|
193
|
+
throw new Error('文件超过 64 MB');
|
|
194
|
+
const extension = extname(original.name).toLowerCase().replace(/[^.a-z0-9]/g, '').slice(0, 16);
|
|
195
|
+
meta.originalName = basename(original.name);
|
|
196
|
+
meta.originalFile = `original${extension || '.bin'}`;
|
|
197
|
+
writeFileSync(join(stage, meta.originalFile), original.bytes);
|
|
198
|
+
}
|
|
199
|
+
const segmentsJson = JSON.stringify(suppliedSegments ?? splitSegments(content), null, 2);
|
|
200
|
+
meta.textChecksum = hash(content);
|
|
201
|
+
meta.segmentsChecksum = hash(segmentsJson);
|
|
202
|
+
writeFileSync(join(stage, 'content.md'), content, 'utf8');
|
|
203
|
+
writeFileSync(join(stage, 'segments.json'), segmentsJson, 'utf8');
|
|
204
|
+
atomicJson(join(stage, 'metadata.json'), meta);
|
|
205
|
+
renameSync(stage, path); // A committed material always has all canonical files.
|
|
206
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
207
|
+
try {
|
|
208
|
+
this.index(meta, path, content);
|
|
209
|
+
this.db.exec('COMMIT');
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
this.db.exec('ROLLBACK');
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
215
|
+
return { material: this.get(id), duplicate: false };
|
|
216
|
+
}
|
|
217
|
+
tags(values) {
|
|
218
|
+
if (values === undefined)
|
|
219
|
+
return [];
|
|
220
|
+
if (!Array.isArray(values) || values.some(v => typeof v !== 'string'))
|
|
221
|
+
throw new Error('标签必须是字符串数组');
|
|
222
|
+
return [...new Set(values.map(v => cleanString(v, 40)).filter(Boolean))].slice(0, 30);
|
|
223
|
+
}
|
|
224
|
+
captureText(text, fields = {}) {
|
|
225
|
+
if (typeof text !== 'string' || !text.trim())
|
|
226
|
+
throw new Error('请填写要保存的正文');
|
|
227
|
+
return this.save(text.trim(), fields, { kind: 'text' });
|
|
228
|
+
}
|
|
229
|
+
captureHtml(html, url, fields = {}) {
|
|
230
|
+
const source = validateUrl(url).href;
|
|
231
|
+
const article = extractArticle(html, source);
|
|
232
|
+
return this.save(article.content, { ...fields, title: fields.title || article.title, sourceUrl: source }, {
|
|
233
|
+
kind: 'web', author: article.author, extraction: 'Readability 正文提取;图片保留远程引用,未离线下载',
|
|
234
|
+
}, { name: 'page.html', bytes: Buffer.from(html) });
|
|
235
|
+
}
|
|
236
|
+
importFile(path, fields = {}) {
|
|
237
|
+
if (!isAbsolute(path))
|
|
238
|
+
throw new Error('请输入文件的绝对路径');
|
|
239
|
+
const info = statSync(path);
|
|
240
|
+
if (!info.isFile() || info.size > MAX_FILE_BYTES)
|
|
241
|
+
throw new Error('仅支持不超过 64 MB 的普通文件');
|
|
242
|
+
return this.importBytes(basename(path), readFileSync(path), fields);
|
|
243
|
+
}
|
|
244
|
+
importBytes(name, bytes, fields = {}) {
|
|
245
|
+
if (bytes.length > MAX_FILE_BYTES || !bytes.length)
|
|
246
|
+
throw new Error('文件不能为空,且不能超过 64 MB');
|
|
247
|
+
const extension = extname(name).toLowerCase();
|
|
248
|
+
const original = { name: basename(name), bytes };
|
|
249
|
+
const data = { ...fields, title: fields.title || basename(name, extension) };
|
|
250
|
+
if (['.vtt', '.srt'].includes(extension)) {
|
|
251
|
+
const segments = parseSubtitles(this.decode(bytes));
|
|
252
|
+
return this.save(segments.map(p => `[${formatTime(p.startSeconds)}] ${p.text}`).join('\n\n'), data, { kind: 'transcript', extraction: '导入字幕;包含时间戳,未分析视频画面' }, original, segments);
|
|
253
|
+
}
|
|
254
|
+
if (['.html', '.htm'].includes(extension)) {
|
|
255
|
+
const article = extractArticle(this.decode(bytes), fields.sourceUrl || 'https://import.invalid/');
|
|
256
|
+
return this.save(article.content, { ...data, title: fields.title || article.title }, { kind: 'file', extraction: '本地 HTML 正文提取', author: article.author }, original);
|
|
257
|
+
}
|
|
258
|
+
if (['.txt', '.md', '.markdown', '.csv', '.json', '.log'].includes(extension)) {
|
|
259
|
+
return this.save(this.decode(bytes), data, { kind: 'file', extraction: 'UTF-8 文本导入' }, original);
|
|
260
|
+
}
|
|
261
|
+
const video = ['.mp4', '.mkv', '.mov', '.webm', '.mp3', '.wav', '.m4a', '.ogg'].includes(extension);
|
|
262
|
+
return this.save('', data, {
|
|
263
|
+
kind: video ? 'video' : 'file', status: 'stored',
|
|
264
|
+
extraction: video ? '已保存原文件;尚未转写,也未分析画面。请另行导入 SRT/VTT 字幕并归入同一专题。' : '已保存原文件;此格式暂不支持正文提取。',
|
|
265
|
+
}, original, []);
|
|
266
|
+
}
|
|
267
|
+
decode(bytes) {
|
|
268
|
+
try {
|
|
269
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
throw new Error('文本编码不是 UTF-8,请转换编码后导入');
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
update(id, fields) {
|
|
276
|
+
const item = this.get(id);
|
|
277
|
+
const { content, segments: _segments, path: oldPath, ...meta } = item;
|
|
278
|
+
if (fields.title !== undefined) {
|
|
279
|
+
if (!cleanString(fields.title))
|
|
280
|
+
throw new Error('标题不能为空');
|
|
281
|
+
meta.title = cleanString(fields.title, 300);
|
|
282
|
+
}
|
|
283
|
+
if (fields.tags !== undefined)
|
|
284
|
+
meta.tags = this.tags(fields.tags);
|
|
285
|
+
if (fields.category !== undefined)
|
|
286
|
+
meta.category = categoryName(fields.category);
|
|
287
|
+
if (fields.collection !== undefined)
|
|
288
|
+
meta.collection = cleanString(fields.collection);
|
|
289
|
+
if (fields.sourceNote !== undefined)
|
|
290
|
+
meta.sourceNote = cleanString(fields.sourceNote, 2000);
|
|
291
|
+
meta.updatedAt = now();
|
|
292
|
+
const path = join(this.root, 'materials', ...meta.category.split('/'), id);
|
|
293
|
+
this.assertContained(path);
|
|
294
|
+
if (path !== oldPath) {
|
|
295
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
296
|
+
renameSync(oldPath, path);
|
|
297
|
+
}
|
|
298
|
+
atomicJson(join(path, 'metadata.json'), meta);
|
|
299
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
300
|
+
try {
|
|
301
|
+
this.index(meta, path, content);
|
|
302
|
+
this.db.exec('COMMIT');
|
|
303
|
+
}
|
|
304
|
+
catch (error) {
|
|
305
|
+
this.db.exec('ROLLBACK');
|
|
306
|
+
throw error;
|
|
307
|
+
}
|
|
308
|
+
return this.get(id);
|
|
309
|
+
}
|
|
310
|
+
saveNote(content, citations, fields) {
|
|
311
|
+
if (!content?.trim())
|
|
312
|
+
throw new Error('笔记内容不能为空');
|
|
313
|
+
if (!Array.isArray(citations) || citations.length > 200)
|
|
314
|
+
throw new Error('引用必须是数组且最多 200 条');
|
|
315
|
+
for (const citation of citations) {
|
|
316
|
+
const source = this.get(citation.materialId);
|
|
317
|
+
if (citation.segmentId && !source.segments.some(s => s.id === citation.segmentId))
|
|
318
|
+
throw new Error(`引用段落不存在:${citation.segmentId}`);
|
|
319
|
+
}
|
|
320
|
+
return this.save(content.trim(), fields, { kind: 'note', citations, extraction: '知识笔记;内容由用户或 Agent 提供,请结合引用核验' });
|
|
321
|
+
}
|
|
322
|
+
search(options = {}) {
|
|
323
|
+
const query = cleanString(options.query, 500);
|
|
324
|
+
const limit = Math.max(1, Math.min(100, Number(options.limit) || 40));
|
|
325
|
+
const offset = Math.max(0, Math.floor(Number(options.offset) || 0));
|
|
326
|
+
const terms = query.toLocaleLowerCase().split(/\s+/).filter(Boolean);
|
|
327
|
+
const rows = this.db.prepare('SELECT id,meta,content FROM materials ORDER BY updated_at DESC').all();
|
|
328
|
+
// Literal substring matching guarantees one- and two-character Chinese searches.
|
|
329
|
+
// FTS narrows body/title candidates for longer terms; metadata is always searched too.
|
|
330
|
+
let indexed;
|
|
331
|
+
if (terms.length === 1 && [...terms[0]].length >= 3) {
|
|
332
|
+
indexed = new Set(this.db.prepare('SELECT id FROM material_fts WHERE material_fts MATCH ?').all(`"${terms[0].replaceAll('"', '""')}"`).map(row => row.id));
|
|
333
|
+
}
|
|
334
|
+
const matched = rows.filter(row => {
|
|
335
|
+
const meta = JSON.parse(row.meta);
|
|
336
|
+
if (options.category && meta.category !== options.category && !meta.category.startsWith(`${options.category}/`))
|
|
337
|
+
return false;
|
|
338
|
+
if (options.kind && meta.kind !== options.kind)
|
|
339
|
+
return false;
|
|
340
|
+
if (options.tag && !meta.tags.includes(options.tag))
|
|
341
|
+
return false;
|
|
342
|
+
if (options.collection && meta.collection !== options.collection)
|
|
343
|
+
return false;
|
|
344
|
+
const metadataText = `${meta.title}\n${meta.category}\n${meta.tags.join(' ')}\n${meta.collection}\n${meta.sourceNote}`.toLocaleLowerCase();
|
|
345
|
+
if (!terms.length)
|
|
346
|
+
return true;
|
|
347
|
+
const haystack = metadataText + '\n' + row.content.toLocaleLowerCase();
|
|
348
|
+
// FTS is an optimization hint only; punctuation and Unicode never change correctness.
|
|
349
|
+
return indexed?.has(row.id) || terms.every(term => haystack.includes(term));
|
|
350
|
+
});
|
|
351
|
+
return {
|
|
352
|
+
total: matched.length,
|
|
353
|
+
items: matched.slice(offset, offset + limit).map(row => {
|
|
354
|
+
const item = this.get(row.id);
|
|
355
|
+
const { content, segments, path: _path, ...meta } = item;
|
|
356
|
+
const matches = terms.length ? segments.filter(p => terms.some(t => p.text.toLocaleLowerCase().includes(t))).slice(0, 3) : [];
|
|
357
|
+
return { ...meta, excerpt: (matches[0]?.text ?? content).slice(0, 240), matches: matches.map(p => ({ ...p, text: p.text.slice(0, 600), citation: `library://${item.id}/${p.id}` })) };
|
|
358
|
+
}),
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
stats() {
|
|
362
|
+
const rows = this.db.prepare('SELECT meta FROM materials').all();
|
|
363
|
+
const materials = rows.map(row => JSON.parse(row.meta));
|
|
364
|
+
const count = (select) => {
|
|
365
|
+
const map = new Map();
|
|
366
|
+
for (const m of materials)
|
|
367
|
+
for (const key of select(m))
|
|
368
|
+
if (key)
|
|
369
|
+
map.set(key, (map.get(key) || 0) + 1);
|
|
370
|
+
return [...map].sort((a, b) => a[0].localeCompare(b[0], 'zh')).map(([name, count]) => ({ name, count }));
|
|
371
|
+
};
|
|
372
|
+
return { total: materials.length, ready: materials.filter(m => m.status === 'ready').length, notes: materials.filter(m => m.kind === 'note').length,
|
|
373
|
+
inbox: materials.filter(m => m.category === '收件箱').length, categories: count(m => [m.category]),
|
|
374
|
+
tags: count(m => m.tags), collections: count(m => [m.collection]), root: this.root, warnings: [...this.warnings] };
|
|
375
|
+
}
|
|
376
|
+
exportMarkdown(id) {
|
|
377
|
+
const item = this.get(id);
|
|
378
|
+
const header = JSON.stringify({ id: item.id, title: item.title, category: item.category, tags: item.tags, collection: item.collection, sourceUrl: item.sourceUrl, createdAt: item.createdAt, revision: item.revision }, null, 2);
|
|
379
|
+
const references = item.citations.map(c => {
|
|
380
|
+
const source = this.get(c.materialId);
|
|
381
|
+
const segment = c.segmentId ? source.segments.find(p => p.id === c.segmentId) : undefined;
|
|
382
|
+
return `- ${source.title} — library://${source.id}${c.segmentId ? `/${c.segmentId}` : ''}${segment?.startSeconds !== undefined ? ` (${formatTime(segment.startSeconds)})` : ''}${source.sourceUrl ? ` — ${source.sourceUrl}` : ''}`;
|
|
383
|
+
}).join('\n');
|
|
384
|
+
return `---\n${header}\n---\n\n# ${item.title}\n\n${item.content}${references ? `\n\n## 来源\n\n${references}` : ''}\n`;
|
|
385
|
+
}
|
|
386
|
+
}
|
package/dist/http.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
import type { Library } from './core/library.js';
|
|
3
|
+
import type { ImportQueue } from './core/jobs.js';
|
|
4
|
+
export declare const API_PATH = "/knowledge-library/api";
|
|
5
|
+
export declare function createHandler(library: Library, queue: ImportQueue, getOrigin: () => string): (req: IncomingMessage, res: ServerResponse) => Promise<void>;
|
|
6
|
+
export declare function serveAsset(req: IncomingMessage, res: ServerResponse): void;
|
package/dist/http.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
export const API_PATH = '/knowledge-library/api';
|
|
4
|
+
function send(res, status, value) {
|
|
5
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' });
|
|
6
|
+
res.end(JSON.stringify(value));
|
|
7
|
+
}
|
|
8
|
+
async function body(req) {
|
|
9
|
+
if (!req.headers['content-type']?.startsWith('application/json'))
|
|
10
|
+
throw new Error('请求必须使用 application/json');
|
|
11
|
+
const chunks = [];
|
|
12
|
+
let length = 0;
|
|
13
|
+
for await (const chunk of req) {
|
|
14
|
+
length += chunk.length;
|
|
15
|
+
if (length > 90 * 1024 * 1024)
|
|
16
|
+
throw new Error('请求体超过限制');
|
|
17
|
+
chunks.push(chunk);
|
|
18
|
+
}
|
|
19
|
+
const data = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
20
|
+
if (!data || typeof data !== 'object' || Array.isArray(data))
|
|
21
|
+
throw new Error('请求内容必须是 JSON 对象');
|
|
22
|
+
return data;
|
|
23
|
+
}
|
|
24
|
+
function fields(data) {
|
|
25
|
+
for (const key of ['title', 'category', 'collection', 'sourceNote', 'sourceUrl']) {
|
|
26
|
+
if (data[key] !== undefined && typeof data[key] !== 'string')
|
|
27
|
+
throw new Error(`${key} 必须是字符串`);
|
|
28
|
+
}
|
|
29
|
+
if (data.tags !== undefined && (!Array.isArray(data.tags) || data.tags.some((tag) => typeof tag !== 'string')))
|
|
30
|
+
throw new Error('标签必须是字符串数组');
|
|
31
|
+
return { title: data.title, category: data.category, collection: data.collection, tags: data.tags, sourceNote: data.sourceNote, sourceUrl: data.sourceUrl };
|
|
32
|
+
}
|
|
33
|
+
const localAddress = (address = '') => address === '::1' || address === '127.0.0.1' || address === '::ffff:127.0.0.1';
|
|
34
|
+
export function createHandler(library, queue, getOrigin) {
|
|
35
|
+
return async (req, res) => {
|
|
36
|
+
try {
|
|
37
|
+
const origin = getOrigin();
|
|
38
|
+
if (!localAddress(req.socket.remoteAddress) || req.headers.host !== new URL(origin).host) {
|
|
39
|
+
send(res, 403, { error: '资料库接口仅允许本机访问' });
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
if (req.headers.origin && req.headers.origin !== origin) {
|
|
43
|
+
send(res, 403, { error: '不允许跨站访问' });
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
const url = new URL(req.url || '/', origin);
|
|
47
|
+
const path = url.pathname.slice(API_PATH.length);
|
|
48
|
+
if (req.method === 'GET') {
|
|
49
|
+
if (path === '/stats') {
|
|
50
|
+
send(res, 200, library.stats());
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (path === '/materials') {
|
|
54
|
+
send(res, 200, library.search({ ...Object.fromEntries(url.searchParams), limit: Number(url.searchParams.get('limit')) || 40, offset: Number(url.searchParams.get('offset')) || 0 }));
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (path === '/jobs') {
|
|
58
|
+
send(res, 200, queue.list());
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const match = path.match(/^\/materials\/(r_[a-f0-9-]+)(?:\/(export|original))?$/);
|
|
62
|
+
if (match) {
|
|
63
|
+
if (match[2] === 'export') {
|
|
64
|
+
res.writeHead(200, { 'Content-Type': 'text/markdown; charset=utf-8', 'Content-Disposition': `attachment; filename="${match[1]}.md"`, 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' });
|
|
65
|
+
res.end(library.exportMarkdown(match[1]));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (match[2] === 'original') {
|
|
69
|
+
const original = library.original(match[1]);
|
|
70
|
+
res.writeHead(200, { 'Content-Type': 'application/octet-stream', 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(original.name).replaceAll("'", '%27')}`, 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-store' });
|
|
71
|
+
res.end(original.bytes);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
send(res, 200, library.get(match[1]));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (req.method === 'POST' || req.method === 'PATCH') {
|
|
79
|
+
if (req.headers['x-knowledge-library'] !== '1') {
|
|
80
|
+
send(res, 403, { error: '缺少同源请求标记' });
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const data = await body(req);
|
|
84
|
+
if (req.method === 'POST' && path === '/capture') {
|
|
85
|
+
const captureFields = fields(data);
|
|
86
|
+
if (data.type === 'url' && typeof data.content === 'string') {
|
|
87
|
+
send(res, 202, { job: queue.add(data.content, captureFields) });
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
if (data.type !== 'text')
|
|
91
|
+
throw new Error('请选择网页或文本');
|
|
92
|
+
send(res, 201, library.captureText(data.content, captureFields));
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (req.method === 'POST' && path === '/upload') {
|
|
96
|
+
if (typeof data.name !== 'string' || typeof data.base64 !== 'string' || !/^[A-Za-z0-9+/]*={0,2}$/.test(data.base64))
|
|
97
|
+
throw new Error('文件数据格式不正确');
|
|
98
|
+
send(res, 201, library.importBytes(data.name, Buffer.from(data.base64, 'base64'), fields(data)));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (req.method === 'POST' && path === '/notes') {
|
|
102
|
+
if (typeof data.content !== 'string' || !Array.isArray(data.citations) || data.citations.some((c) => !c || typeof c.materialId !== 'string' || (c.segmentId !== undefined && typeof c.segmentId !== 'string')))
|
|
103
|
+
throw new Error('笔记内容或引用格式不正确');
|
|
104
|
+
send(res, 201, library.saveNote(data.content, data.citations, fields(data)));
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const match = path.match(/^\/materials\/(r_[a-f0-9-]+)$/);
|
|
108
|
+
if (req.method === 'PATCH' && match) {
|
|
109
|
+
send(res, 200, library.update(match[1], fields(data)));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const retry = path.match(/^\/jobs\/(job_[a-f0-9-]+)\/retry$/);
|
|
113
|
+
if (req.method === 'POST' && retry) {
|
|
114
|
+
send(res, 200, queue.retry(retry[1]));
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (req.method === 'POST' && path === '/rebuild') {
|
|
118
|
+
send(res, 200, library.rebuild());
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
send(res, 404, { error: '接口不存在' });
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
if (!res.headersSent)
|
|
126
|
+
send(res, 400, { error: error instanceof Error ? error.message : '操作失败' });
|
|
127
|
+
else
|
|
128
|
+
res.end();
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
export function serveAsset(req, res) {
|
|
133
|
+
const path = new URL(req.url || '/', 'http://localhost').pathname;
|
|
134
|
+
const names = {
|
|
135
|
+
'/': ['index.html', 'text/html; charset=utf-8'],
|
|
136
|
+
'/app.js': ['app.js', 'text/javascript; charset=utf-8'],
|
|
137
|
+
'/app.css': ['app.css', 'text/css; charset=utf-8'],
|
|
138
|
+
};
|
|
139
|
+
const item = names[path];
|
|
140
|
+
if (!item) {
|
|
141
|
+
res.writeHead(404);
|
|
142
|
+
res.end();
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
const bytes = readFileSync(fileURLToPath(new URL(`../dist/public/${item[0]}`, import.meta.url)));
|
|
147
|
+
res.writeHead(200, {
|
|
148
|
+
'Content-Type': item[1], 'X-Content-Type-Options': 'nosniff',
|
|
149
|
+
'Content-Security-Policy': "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; object-src 'none'; base-uri 'none'",
|
|
150
|
+
});
|
|
151
|
+
res.end(bytes);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
res.writeHead(503);
|
|
155
|
+
res.end('请先运行 npm run build');
|
|
156
|
+
}
|
|
157
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import z from '@deepseek-ai/schemastery';
|
|
3
|
+
export declare const name = "knowledge-library";
|
|
4
|
+
export declare const inject: string[];
|
|
5
|
+
export interface Config {
|
|
6
|
+
root: string;
|
|
7
|
+
allowProxyFakeIp?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
10
|
+
root: z<string, string, "defined">;
|
|
11
|
+
allowProxyFakeIp: z<boolean, boolean, "defined">;
|
|
12
|
+
}>>, Schemastery.ObjectT<NoInfer<{
|
|
13
|
+
root: z<string, string, "defined">;
|
|
14
|
+
allowProxyFakeIp: z<boolean, boolean, "defined">;
|
|
15
|
+
}>>, "plain">;
|
|
16
|
+
export declare function apply(ctx: Context, config: Config): void;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import z from '@deepseek-ai/schemastery';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { Library } from './core/library.js';
|
|
5
|
+
import { ImportQueue } from './core/jobs.js';
|
|
6
|
+
import { createTools } from './tools.js';
|
|
7
|
+
import { API_PATH, createHandler } from './http.js';
|
|
8
|
+
import { fetchPublicPage } from './core/fetch.js';
|
|
9
|
+
export const name = 'knowledge-library';
|
|
10
|
+
export const inject = ['tools'];
|
|
11
|
+
export const Config = z.object({
|
|
12
|
+
root: z.string().default(join(homedir(), 'Documents', 'DSH-Knowledge-Library')).description('资料库绝对路径;更改后重新加载插件。资料独立于聊天和插件安装目录。'),
|
|
13
|
+
allowProxyFakeIp: z.boolean().default(false).description('仅在本机使用 Clash/Mihomo TUN fake-IP DNS 时启用。允许公网域名解析到 198.18.0.0/15,仍拒绝本机和其他内网地址。'),
|
|
14
|
+
});
|
|
15
|
+
export function apply(ctx, config) {
|
|
16
|
+
const library = new Library(config.root);
|
|
17
|
+
let queue;
|
|
18
|
+
try {
|
|
19
|
+
queue = new ImportQueue(library, (url, signal) => fetchPublicPage(url, signal, { allowProxyFakeIp: config.allowProxyFakeIp }));
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
library.close();
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
ctx.effect(() => async () => { await queue.close(); library.close(); });
|
|
26
|
+
for (const tool of createTools(library, queue))
|
|
27
|
+
ctx.tools.register(tool);
|
|
28
|
+
ctx.inject(['webServer'], child => {
|
|
29
|
+
child.webServer.register({
|
|
30
|
+
kind: 'prefix', path: API_PATH,
|
|
31
|
+
handler: createHandler(library, queue, () => child.webServer.getOrigin()),
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
ctx.logger.info(`资料库已加载:${library.root}`);
|
|
35
|
+
}
|