dsh-knowledge-library 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +123 -18
- package/dist/client.js +1699 -374
- package/dist/core/extract.d.ts +1 -0
- package/dist/core/extract.js +62 -5
- package/dist/core/fetch.js +82 -11
- package/dist/core/grouping.d.ts +6 -0
- package/dist/core/grouping.js +71 -0
- package/dist/core/incremental.d.ts +14 -0
- package/dist/core/incremental.js +52 -0
- package/dist/core/jobs.d.ts +1 -0
- package/dist/core/jobs.js +4 -0
- package/dist/core/knowledge.d.ts +37 -0
- package/dist/core/knowledge.js +617 -0
- package/dist/core/library.d.ts +20 -2
- package/dist/core/library.js +271 -12
- package/dist/core/page-access.d.ts +6 -0
- package/dist/core/page-access.js +38 -0
- package/dist/core/purge.d.ts +20 -0
- package/dist/core/purge.js +106 -0
- package/dist/directory-settings.d.ts +18 -0
- package/dist/directory-settings.js +92 -0
- package/dist/http.d.ts +3 -1
- package/dist/http.js +121 -4
- package/dist/index.js +17 -5
- package/dist/knowledge-model.d.ts +18 -0
- package/dist/knowledge-model.js +116 -0
- package/dist/preview-settings.d.ts +6 -0
- package/dist/preview-settings.js +42 -0
- package/dist/public/app.css +385 -182
- package/dist/public/app.js +11 -7
- package/dist/public/index.html +1 -1
- package/dist/standalone.js +8 -3
- package/dist/tools.d.ts +2 -1
- package/dist/tools.js +10 -1
- package/dist/types.d.ts +143 -1
- package/dist/ui/App.js +244 -47
- package/dist/ui/Dialog.d.ts +7 -0
- package/dist/ui/Dialog.js +32 -0
- package/dist/ui/Grouping.d.ts +12 -0
- package/dist/ui/Grouping.js +93 -0
- package/dist/ui/Knowledge.d.ts +26 -0
- package/dist/ui/Knowledge.js +106 -0
- package/dist/ui/api.d.ts +7 -0
- package/dist/ui/api.js +16 -0
- package/dist/ui/icons.js +2 -0
- package/package.json +4 -1
package/dist/core/library.js
CHANGED
|
@@ -2,8 +2,10 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
2
2
|
import { randomUUID, createHash } from 'node:crypto';
|
|
3
3
|
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, readdirSync, statSync, lstatSync, unlinkSync, openSync, closeSync } from 'node:fs';
|
|
4
4
|
import { resolve, join, dirname, basename, extname, relative, isAbsolute } from 'node:path';
|
|
5
|
+
import { PurgeStore } from './purge.js';
|
|
5
6
|
import { extractArticle, formatTime, parseSubtitles, splitSegments } from './extract.js';
|
|
6
7
|
import { validateUrl } from './fetch.js';
|
|
8
|
+
import { noteFingerprint } from './incremental.js';
|
|
7
9
|
export const MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
8
10
|
export const MAX_TEXT_CHARS = 2_000_000;
|
|
9
11
|
const hash = (data) => createHash('sha256').update(data).digest('hex');
|
|
@@ -21,6 +23,11 @@ export function atomicJson(path, value) {
|
|
|
21
23
|
writeFileSync(tmp, JSON.stringify(value, null, 2), { encoding: 'utf8', flag: 'wx' });
|
|
22
24
|
renameSync(tmp, path);
|
|
23
25
|
}
|
|
26
|
+
function atomicText(path, value) {
|
|
27
|
+
const tmp = `${path}.${randomUUID()}.tmp`;
|
|
28
|
+
writeFileSync(tmp, value, { encoding: 'utf8', flag: 'wx' });
|
|
29
|
+
renameSync(tmp, path);
|
|
30
|
+
}
|
|
24
31
|
export class Library {
|
|
25
32
|
root;
|
|
26
33
|
db;
|
|
@@ -59,7 +66,9 @@ export class Library {
|
|
|
59
66
|
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
|
|
60
67
|
CREATE TABLE IF NOT EXISTS materials (id TEXT PRIMARY KEY, path TEXT NOT NULL, meta TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, updated_at TEXT NOT NULL);
|
|
61
68
|
CREATE VIRTUAL TABLE IF NOT EXISTS material_fts USING fts5(id UNINDEXED, title, content, tokenize='trigram');`);
|
|
69
|
+
this.recoverNoteUpdate();
|
|
62
70
|
this.rebuild();
|
|
71
|
+
this.warnings.push(...this.cleanupPendingPurge().map(error => `待清理文件:${error}`));
|
|
63
72
|
}
|
|
64
73
|
catch (error) {
|
|
65
74
|
try {
|
|
@@ -95,6 +104,7 @@ export class Library {
|
|
|
95
104
|
this.db.prepare('INSERT INTO material_fts (id,title,content) VALUES (?,?,?)').run(meta.id, meta.title, content);
|
|
96
105
|
}
|
|
97
106
|
rebuild() {
|
|
107
|
+
this.recoverNoteUpdate();
|
|
98
108
|
this.warnings.length = 0;
|
|
99
109
|
const entries = [];
|
|
100
110
|
const visit = (directory, depth) => {
|
|
@@ -154,9 +164,11 @@ export class Library {
|
|
|
154
164
|
return row;
|
|
155
165
|
}
|
|
156
166
|
get(id) {
|
|
167
|
+
this.recoverNoteUpdate();
|
|
157
168
|
const row = this.row(id);
|
|
158
169
|
return { ...JSON.parse(row.meta), content: row.content, segments: JSON.parse(readFileSync(join(row.path, 'segments.json'), 'utf8')), path: row.path };
|
|
159
170
|
}
|
|
171
|
+
has(id) { return Boolean(this.db.prepare('SELECT id FROM materials WHERE id=?').get(id)); }
|
|
160
172
|
original(id) {
|
|
161
173
|
const item = this.get(id);
|
|
162
174
|
if (!item.originalFile || basename(item.originalFile) !== item.originalFile)
|
|
@@ -166,12 +178,13 @@ export class Library {
|
|
|
166
178
|
return { bytes: readFileSync(path), name: item.originalName || item.originalFile };
|
|
167
179
|
}
|
|
168
180
|
save(content, fields, extra, original, suppliedSegments) {
|
|
181
|
+
this.recoverNoteUpdate();
|
|
169
182
|
if (content.length > MAX_TEXT_CHARS)
|
|
170
183
|
throw new Error('正文超过 200 万字符,请拆分后导入');
|
|
171
|
-
const digest = hash(`${extra.kind}\0${original ? hash(original.bytes) : content}\0${fields.sourceUrl ?? ''}\0${JSON.stringify(extra.citations ?? [])}`);
|
|
184
|
+
const digest = hash(`${extra.kind}\0${original ? hash(original.bytes) : content}\0${fields.sourceUrl ?? ''}\0${JSON.stringify(extra.citations ?? [])}${extra.knowledgeJobId ? `\0${extra.knowledgeJobId}` : ''}`);
|
|
172
185
|
const duplicate = this.db.prepare('SELECT id FROM materials WHERE content_hash=? LIMIT 1').get(digest);
|
|
173
186
|
if (duplicate)
|
|
174
|
-
return { material: this.get(duplicate.id), duplicate: true };
|
|
187
|
+
return { material: this.get(duplicate.id).deletedAt ? this.restore(duplicate.id) : this.get(duplicate.id), duplicate: true };
|
|
175
188
|
const category = categoryName(fields.category);
|
|
176
189
|
const id = `r_${randomUUID()}`;
|
|
177
190
|
const timestamp = now();
|
|
@@ -230,7 +243,7 @@ export class Library {
|
|
|
230
243
|
const source = validateUrl(url).href;
|
|
231
244
|
const article = extractArticle(html, source);
|
|
232
245
|
return this.save(article.content, { ...fields, title: fields.title || article.title, sourceUrl: source }, {
|
|
233
|
-
kind: 'web', author: article.author, extraction:
|
|
246
|
+
kind: 'web', author: article.author, extraction: `${article.extraction};图片保留远程引用,未离线下载`,
|
|
234
247
|
}, { name: 'page.html', bytes: Buffer.from(html) });
|
|
235
248
|
}
|
|
236
249
|
importFile(path, fields = {}) {
|
|
@@ -253,7 +266,7 @@ export class Library {
|
|
|
253
266
|
}
|
|
254
267
|
if (['.html', '.htm'].includes(extension)) {
|
|
255
268
|
const article = extractArticle(this.decode(bytes), fields.sourceUrl || 'https://import.invalid/');
|
|
256
|
-
return this.save(article.content, { ...data, title: fields.title || article.title }, { kind: 'file', extraction:
|
|
269
|
+
return this.save(article.content, { ...data, title: fields.title || article.title }, { kind: 'file', extraction: `本地 HTML:${article.extraction};图片保留远程引用,未离线下载`, author: article.author }, original);
|
|
257
270
|
}
|
|
258
271
|
if (['.txt', '.md', '.markdown', '.csv', '.json', '.log'].includes(extension)) {
|
|
259
272
|
return this.save(this.decode(bytes), data, { kind: 'file', extraction: 'UTF-8 文本导入' }, original);
|
|
@@ -274,7 +287,10 @@ export class Library {
|
|
|
274
287
|
}
|
|
275
288
|
update(id, fields) {
|
|
276
289
|
const item = this.get(id);
|
|
290
|
+
if (item.deletedAt)
|
|
291
|
+
throw new Error('请先从回收站恢复资料后再编辑');
|
|
277
292
|
const { content, segments: _segments, path: oldPath, ...meta } = item;
|
|
293
|
+
const previous = structuredClone(meta);
|
|
278
294
|
if (fields.title !== undefined) {
|
|
279
295
|
if (!cleanString(fields.title))
|
|
280
296
|
throw new Error('标题不能为空');
|
|
@@ -291,23 +307,148 @@ export class Library {
|
|
|
291
307
|
meta.updatedAt = now();
|
|
292
308
|
const path = join(this.root, 'materials', ...meta.category.split('/'), id);
|
|
293
309
|
this.assertContained(path);
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
310
|
+
let moved = false;
|
|
311
|
+
let written = false;
|
|
312
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
313
|
+
try {
|
|
314
|
+
if (path !== oldPath) {
|
|
315
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
316
|
+
renameSync(oldPath, path);
|
|
317
|
+
moved = true;
|
|
318
|
+
}
|
|
319
|
+
atomicJson(join(path, 'metadata.json'), meta);
|
|
320
|
+
written = true;
|
|
321
|
+
this.index(meta, path, content);
|
|
322
|
+
this.db.exec('COMMIT');
|
|
323
|
+
}
|
|
324
|
+
catch (error) {
|
|
325
|
+
this.db.exec('ROLLBACK');
|
|
326
|
+
if (written)
|
|
327
|
+
atomicJson(join(path, 'metadata.json'), previous);
|
|
328
|
+
if (moved)
|
|
329
|
+
renameSync(path, oldPath);
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
return this.get(id);
|
|
333
|
+
}
|
|
334
|
+
validateCategoryMove(id, category) {
|
|
335
|
+
const source = this.get(id);
|
|
336
|
+
const target = join(this.root, 'materials', ...categoryName(category).split('/'), id);
|
|
337
|
+
this.assertContained(target);
|
|
338
|
+
if (resolve(target) !== resolve(source.path) && existsSync(target))
|
|
339
|
+
throw new Error('目标分类已有同ID目录,请检查后重试');
|
|
340
|
+
}
|
|
341
|
+
trash(id) { return this.setDeleted(id, true); }
|
|
342
|
+
restore(id) { return this.setDeleted(id, false); }
|
|
343
|
+
setDeleted(id, deleted) {
|
|
344
|
+
const item = this.get(id);
|
|
345
|
+
if (Boolean(item.deletedAt) === deleted)
|
|
346
|
+
return item;
|
|
347
|
+
const { content, segments: _segments, path, ...previous } = item;
|
|
348
|
+
const meta = { ...previous, updatedAt: now() };
|
|
349
|
+
if (deleted) {
|
|
350
|
+
meta.deletedAt = now();
|
|
351
|
+
meta.deletionId = randomUUID();
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
delete meta.deletedAt;
|
|
355
|
+
delete meta.deletionId;
|
|
297
356
|
}
|
|
298
|
-
|
|
357
|
+
// Keep the canonical files and stable IDs in place so existing citations
|
|
358
|
+
// remain readable. The flag also survives rebuilding a lost SQLite index.
|
|
299
359
|
this.db.exec('BEGIN IMMEDIATE');
|
|
360
|
+
let written = false;
|
|
300
361
|
try {
|
|
362
|
+
atomicJson(join(path, 'metadata.json'), meta);
|
|
363
|
+
written = true;
|
|
301
364
|
this.index(meta, path, content);
|
|
302
365
|
this.db.exec('COMMIT');
|
|
303
366
|
}
|
|
304
367
|
catch (error) {
|
|
305
368
|
this.db.exec('ROLLBACK');
|
|
369
|
+
if (written)
|
|
370
|
+
atomicJson(join(path, 'metadata.json'), previous);
|
|
306
371
|
throw error;
|
|
307
372
|
}
|
|
308
373
|
return this.get(id);
|
|
309
374
|
}
|
|
310
|
-
|
|
375
|
+
trashPlan() {
|
|
376
|
+
const rows = this.db.prepare('SELECT id,path,meta FROM materials ORDER BY id').all();
|
|
377
|
+
const items = rows.map(row => ({ ...row, metadata: JSON.parse(row.meta) })).filter(row => row.metadata.deletedAt);
|
|
378
|
+
const pending = new PurgeStore(this.root).list();
|
|
379
|
+
const token = hash(JSON.stringify({ items: items.map(row => [row.id, row.metadata.deletedAt, row.metadata.deletionId]), pending: pending.map(row => row.id) }));
|
|
380
|
+
return { items, pending, token };
|
|
381
|
+
}
|
|
382
|
+
trashSnapshot() {
|
|
383
|
+
const plan = this.trashPlan();
|
|
384
|
+
return { count: plan.items.length, pending: plan.pending.length, token: plan.token };
|
|
385
|
+
}
|
|
386
|
+
cleanupPendingPurge() {
|
|
387
|
+
const store = new PurgeStore(this.root);
|
|
388
|
+
const errors = [];
|
|
389
|
+
try {
|
|
390
|
+
for (const ticket of store.list()) {
|
|
391
|
+
try {
|
|
392
|
+
store.finish(ticket);
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
errors.push(`${ticket.id}:${error.message}`);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
catch (error) {
|
|
400
|
+
errors.push(error.message);
|
|
401
|
+
}
|
|
402
|
+
return errors;
|
|
403
|
+
}
|
|
404
|
+
emptyTrash(expectedToken) {
|
|
405
|
+
this.recoverNoteUpdate();
|
|
406
|
+
const plan = this.trashPlan();
|
|
407
|
+
if (typeof expectedToken !== 'string' || expectedToken !== plan.token)
|
|
408
|
+
throw new Error('回收站内容已改变,请重新打开清空确认窗口');
|
|
409
|
+
const store = new PurgeStore(this.root);
|
|
410
|
+
// Preflight the entire confirmed scope before moving or removing anything.
|
|
411
|
+
for (const item of plan.items) {
|
|
412
|
+
const expected = join(this.root, 'materials', ...categoryName(item.metadata.category).split('/'), item.id);
|
|
413
|
+
if (resolve(item.path) !== resolve(expected))
|
|
414
|
+
throw new Error('清理目标与资料目录不一致');
|
|
415
|
+
store.validate(item.id, item.path);
|
|
416
|
+
}
|
|
417
|
+
const errors = this.cleanupPendingPurge();
|
|
418
|
+
let deleted = 0;
|
|
419
|
+
for (const item of plan.items) {
|
|
420
|
+
try {
|
|
421
|
+
const ticket = store.stage(item.id, item.path);
|
|
422
|
+
let transaction = false;
|
|
423
|
+
try {
|
|
424
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
425
|
+
transaction = true;
|
|
426
|
+
this.db.prepare('DELETE FROM material_fts WHERE id=?').run(item.id);
|
|
427
|
+
this.db.prepare('DELETE FROM materials WHERE id=?').run(item.id);
|
|
428
|
+
this.db.exec('COMMIT');
|
|
429
|
+
transaction = false;
|
|
430
|
+
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
if (transaction)
|
|
433
|
+
this.db.exec('ROLLBACK');
|
|
434
|
+
store.rollback(ticket);
|
|
435
|
+
throw error;
|
|
436
|
+
}
|
|
437
|
+
deleted++;
|
|
438
|
+
try {
|
|
439
|
+
store.finish(ticket);
|
|
440
|
+
}
|
|
441
|
+
catch (error) {
|
|
442
|
+
errors.push(`${item.id}:文件清理未完成,${error.message}`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
errors.push(`${item.id}:${error.message}`);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return { deleted, pending: store.list().length, errors };
|
|
450
|
+
}
|
|
451
|
+
saveNote(content, citations, fields, knowledgeJobId) {
|
|
311
452
|
if (!content?.trim())
|
|
312
453
|
throw new Error('笔记内容不能为空');
|
|
313
454
|
if (!Array.isArray(citations) || citations.length > 200)
|
|
@@ -317,9 +458,116 @@ export class Library {
|
|
|
317
458
|
if (citation.segmentId && !source.segments.some(s => s.id === citation.segmentId))
|
|
318
459
|
throw new Error(`引用段落不存在:${citation.segmentId}`);
|
|
319
460
|
}
|
|
320
|
-
return this.save(content.trim(), fields, { kind: 'note', citations, extraction: '知识笔记;内容由用户或 Agent 提供,请结合引用核验' });
|
|
461
|
+
return this.save(content.trim(), fields, { kind: 'note', citations, ...(knowledgeJobId ? { knowledgeJobId } : {}), extraction: '知识笔记;内容由用户或 Agent 提供,请结合引用核验' });
|
|
462
|
+
}
|
|
463
|
+
writeNoteSnapshot(snapshot) {
|
|
464
|
+
const path = join(this.root, 'materials', ...categoryName(snapshot.meta.category).split('/'), snapshot.meta.id);
|
|
465
|
+
for (const file of ['content.md', 'segments.json', 'metadata.json'])
|
|
466
|
+
this.assertContained(join(path, file));
|
|
467
|
+
atomicText(join(path, 'content.md'), snapshot.content);
|
|
468
|
+
atomicJson(join(path, 'segments.json'), snapshot.segments);
|
|
469
|
+
atomicJson(join(path, 'metadata.json'), snapshot.meta);
|
|
470
|
+
return path;
|
|
471
|
+
}
|
|
472
|
+
archiveNoteUpdate(update) {
|
|
473
|
+
const history = join(this.root, 'materials', ...categoryName(update.before.meta.category).split('/'), update.before.meta.id, '.history');
|
|
474
|
+
this.assertContained(history);
|
|
475
|
+
mkdirSync(history, { recursive: true });
|
|
476
|
+
const target = join(history, `${update.jobId}.json`);
|
|
477
|
+
this.assertContained(target);
|
|
478
|
+
renameSync(join(this.root, '.system', 'note-update.json'), target);
|
|
479
|
+
}
|
|
480
|
+
recoverNoteUpdate() {
|
|
481
|
+
const file = join(this.root, '.system', 'note-update.json');
|
|
482
|
+
this.assertContained(file);
|
|
483
|
+
if (!existsSync(file))
|
|
484
|
+
return;
|
|
485
|
+
const update = JSON.parse(readFileSync(file, 'utf8'));
|
|
486
|
+
if (!update || !/^knowledge_[a-f0-9-]+$/.test(update.jobId) || !['prepared', 'committed'].includes(update.phase))
|
|
487
|
+
throw new Error('笔记更新日志损坏,请检查 .system/note-update.json');
|
|
488
|
+
for (const snapshot of [update.before, update.after]) {
|
|
489
|
+
if (!snapshot || snapshot.meta?.kind !== 'note' || !/^r_[a-f0-9-]+$/.test(snapshot.meta.id) || typeof snapshot.content !== 'string' || !Array.isArray(snapshot.segments) || hash(snapshot.content) !== snapshot.meta.textChecksum || hash(JSON.stringify(snapshot.segments, null, 2)) !== snapshot.meta.segmentsChecksum)
|
|
490
|
+
throw new Error('笔记更新快照损坏,请检查更新日志');
|
|
491
|
+
}
|
|
492
|
+
if (update.before.meta.id !== update.after.meta.id || update.before.meta.category !== update.after.meta.category || update.after.meta.revision !== update.before.meta.revision + 1)
|
|
493
|
+
throw new Error('笔记更新版本无效');
|
|
494
|
+
const snapshot = update.phase === 'committed' ? update.after : update.before;
|
|
495
|
+
const path = this.writeNoteSnapshot(snapshot);
|
|
496
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
497
|
+
try {
|
|
498
|
+
this.index(snapshot.meta, path, snapshot.content);
|
|
499
|
+
this.db.exec('COMMIT');
|
|
500
|
+
}
|
|
501
|
+
catch (error) {
|
|
502
|
+
this.db.exec('ROLLBACK');
|
|
503
|
+
throw error;
|
|
504
|
+
}
|
|
505
|
+
if (update.phase === 'committed')
|
|
506
|
+
this.archiveNoteUpdate(update);
|
|
507
|
+
else
|
|
508
|
+
unlinkSync(file);
|
|
509
|
+
}
|
|
510
|
+
/** Append only after explicit confirmation. Existing paragraph IDs and text never change. */
|
|
511
|
+
appendNote(id, expectedFingerprint, addition, citations, tags, jobId) {
|
|
512
|
+
this.recoverNoteUpdate();
|
|
513
|
+
const previous = this.generatedNote(jobId);
|
|
514
|
+
if (previous)
|
|
515
|
+
return { material: previous, duplicate: true };
|
|
516
|
+
const item = this.get(id);
|
|
517
|
+
if (item.kind !== 'note' || item.deletedAt)
|
|
518
|
+
throw new Error('目标知识笔记已删除或不可用,请改为新建或重新整理');
|
|
519
|
+
if (noteFingerprint(item) !== expectedFingerprint)
|
|
520
|
+
throw new Error('已有笔记已改变,请改为新建,或重新整理后核对');
|
|
521
|
+
if (!/^knowledge_[a-f0-9-]+$/.test(jobId) || typeof addition !== 'string' || addition.length > 50_000)
|
|
522
|
+
throw new Error('笔记补充参数无效');
|
|
523
|
+
for (const citation of citations) {
|
|
524
|
+
const source = this.get(citation.materialId);
|
|
525
|
+
if (source.deletedAt || source.kind === 'note' || !citation.segmentId || !source.segments.some(s => s.id === citation.segmentId))
|
|
526
|
+
throw new Error('新增来源已删除或引用无效');
|
|
527
|
+
}
|
|
528
|
+
const merged = [...new Map([...item.citations, ...citations].map(c => [JSON.stringify([c.materialId, c.segmentId]), c])).values()];
|
|
529
|
+
if (merged.length > 200)
|
|
530
|
+
throw new Error('合并后来源超过200条,请改为新建笔记');
|
|
531
|
+
const mergedTags = [...new Set([...item.tags, ...this.tags(tags)])];
|
|
532
|
+
if (mergedTags.length > 30)
|
|
533
|
+
throw new Error('合并后标签超过30个,请减少标签或新建笔记');
|
|
534
|
+
const suffix = addition.trim() ? `## 知识补充 · ${now().slice(0, 10)}\n\n${addition.trim()}` : '';
|
|
535
|
+
const content = item.content + (suffix ? `\n\n${suffix}` : '');
|
|
536
|
+
if (content.length > MAX_TEXT_CHARS)
|
|
537
|
+
throw new Error('笔记超过正文上限,请改为新建');
|
|
538
|
+
const segments = [...item.segments, ...splitSegments(suffix).map(s => ({ ...s, id: `u_${randomUUID().replaceAll('-', '')}` }))];
|
|
539
|
+
const { content: oldContent, segments: oldSegments, path: _path, ...meta } = item;
|
|
540
|
+
const before = { meta, content: oldContent, segments: oldSegments };
|
|
541
|
+
const after = { content, segments, meta: { ...meta, revision: meta.revision + 1, updatedAt: now(), tags: mergedTags, citations: merged, knowledgeUpdateJobIds: [...(meta.knowledgeUpdateJobIds || []), jobId], contentHash: hash(`note-update\0${content}\0${jobId}`), textChecksum: hash(content), segmentsChecksum: hash(JSON.stringify(segments, null, 2)) } };
|
|
542
|
+
const update = { jobId, phase: 'prepared', before, after };
|
|
543
|
+
const journal = join(this.root, '.system', 'note-update.json');
|
|
544
|
+
this.assertContained(journal);
|
|
545
|
+
atomicJson(journal, update);
|
|
546
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
547
|
+
try {
|
|
548
|
+
const path = this.writeNoteSnapshot(after);
|
|
549
|
+
this.index(after.meta, path, content);
|
|
550
|
+
update.phase = 'committed';
|
|
551
|
+
atomicJson(journal, update);
|
|
552
|
+
this.db.exec('COMMIT');
|
|
553
|
+
}
|
|
554
|
+
catch (error) {
|
|
555
|
+
this.db.exec('ROLLBACK');
|
|
556
|
+
update.phase = 'prepared';
|
|
557
|
+
atomicJson(journal, update);
|
|
558
|
+
this.writeNoteSnapshot(before);
|
|
559
|
+
unlinkSync(journal);
|
|
560
|
+
throw error;
|
|
561
|
+
}
|
|
562
|
+
this.archiveNoteUpdate(update);
|
|
563
|
+
return { material: this.get(id), duplicate: false };
|
|
564
|
+
}
|
|
565
|
+
generatedNote(jobId) {
|
|
566
|
+
const row = this.db.prepare("SELECT id FROM materials WHERE json_extract(meta, '$.knowledgeJobId')=? OR EXISTS (SELECT 1 FROM json_each(materials.meta, '$.knowledgeUpdateJobIds') WHERE value=?) LIMIT 1").get(jobId, jobId);
|
|
567
|
+
return row ? this.get(row.id) : undefined;
|
|
321
568
|
}
|
|
322
569
|
search(options = {}) {
|
|
570
|
+
this.recoverNoteUpdate();
|
|
323
571
|
const query = cleanString(options.query, 500);
|
|
324
572
|
const limit = Math.max(1, Math.min(100, Number(options.limit) || 40));
|
|
325
573
|
const offset = Math.max(0, Math.floor(Number(options.offset) || 0));
|
|
@@ -333,6 +581,8 @@ export class Library {
|
|
|
333
581
|
}
|
|
334
582
|
const matched = rows.filter(row => {
|
|
335
583
|
const meta = JSON.parse(row.meta);
|
|
584
|
+
if (Boolean(meta.deletedAt) !== (options.trashed === true))
|
|
585
|
+
return false;
|
|
336
586
|
if (options.category && meta.category !== options.category && !meta.category.startsWith(`${options.category}/`))
|
|
337
587
|
return false;
|
|
338
588
|
if (options.kind && meta.kind !== options.kind)
|
|
@@ -359,8 +609,10 @@ export class Library {
|
|
|
359
609
|
};
|
|
360
610
|
}
|
|
361
611
|
stats() {
|
|
612
|
+
this.recoverNoteUpdate();
|
|
362
613
|
const rows = this.db.prepare('SELECT meta FROM materials').all();
|
|
363
|
-
const
|
|
614
|
+
const all = rows.map(row => JSON.parse(row.meta));
|
|
615
|
+
const materials = all.filter(item => !item.deletedAt);
|
|
364
616
|
const count = (select) => {
|
|
365
617
|
const map = new Map();
|
|
366
618
|
for (const m of materials)
|
|
@@ -369,7 +621,12 @@ export class Library {
|
|
|
369
621
|
map.set(key, (map.get(key) || 0) + 1);
|
|
370
622
|
return [...map].sort((a, b) => a[0].localeCompare(b[0], 'zh')).map(([name, count]) => ({ name, count }));
|
|
371
623
|
};
|
|
372
|
-
|
|
624
|
+
let purgePending = 0;
|
|
625
|
+
try {
|
|
626
|
+
purgePending = new PurgeStore(this.root).list().length;
|
|
627
|
+
}
|
|
628
|
+
catch { /* Clearing reports invalid cleanup records without breaking normal reading. */ }
|
|
629
|
+
return { total: materials.length, trashed: all.length - materials.length, canEmptyTrash: true, purgePending, ready: materials.filter(m => m.status === 'ready').length, notes: materials.filter(m => m.kind === 'note').length,
|
|
373
630
|
inbox: materials.filter(m => m.category === '收件箱').length, categories: count(m => [m.category]),
|
|
374
631
|
tags: count(m => m.tags), collections: count(m => [m.collection]), root: this.root, warnings: [...this.warnings] };
|
|
375
632
|
}
|
|
@@ -377,6 +634,8 @@ export class Library {
|
|
|
377
634
|
const item = this.get(id);
|
|
378
635
|
const header = JSON.stringify({ id: item.id, title: item.title, category: item.category, tags: item.tags, collection: item.collection, sourceUrl: item.sourceUrl, createdAt: item.createdAt, revision: item.revision }, null, 2);
|
|
379
636
|
const references = item.citations.map(c => {
|
|
637
|
+
if (!this.has(c.materialId))
|
|
638
|
+
return `- 来源已永久删除 — library://${c.materialId}${c.segmentId ? `/${c.segmentId}` : ''}`;
|
|
380
639
|
const source = this.get(c.materialId);
|
|
381
640
|
const segment = c.segmentId ? source.segments.find(p => p.id === c.segmentId) : undefined;
|
|
382
641
|
return `- ${source.title} — library://${source.id}${c.segmentId ? `/${c.segmentId}` : ''}${segment?.startSeconds !== undefined ? ` (${formatTime(segment.startSeconds)})` : ''}${source.sourceUrl ? ` — ${source.sourceUrl}` : ''}`;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare class PageAccessError extends Error {
|
|
2
|
+
readonly code = "PAGE_REQUIRES_BROWSER";
|
|
3
|
+
constructor(site: string);
|
|
4
|
+
}
|
|
5
|
+
/** Match explicit challenge structure, not keywords occurring in an ordinary article. */
|
|
6
|
+
export declare function assertArticleAccess(document: Document, source: string): void;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export class PageAccessError extends Error {
|
|
2
|
+
code = 'PAGE_REQUIRES_BROWSER';
|
|
3
|
+
constructor(site) {
|
|
4
|
+
super(`${site}返回了验证页面,尚未获取文章正文。请在浏览器打开原网页,完成验证后粘贴正文,或导入保存的 HTML。后台采集不共享浏览器的验证状态。`);
|
|
5
|
+
this.name = 'PageAccessError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/** Match explicit challenge structure, not keywords occurring in an ordinary article. */
|
|
9
|
+
export function assertArticleAccess(document, source) {
|
|
10
|
+
const url = new URL(source);
|
|
11
|
+
const host = url.hostname.toLowerCase().replace(/\.$/, '');
|
|
12
|
+
const text = (document.body?.textContent || '').replace(/\s+/g, ' ').trim();
|
|
13
|
+
const title = document.title?.trim() || '';
|
|
14
|
+
const heading = document.querySelector('h1,h2,.weui-msg__title')?.textContent?.trim() || '';
|
|
15
|
+
const verificationLink = document.querySelector('a[href*="/mp/wappoc_appmsgcaptcha"]');
|
|
16
|
+
let importedWechatChallenge = false;
|
|
17
|
+
if (host === 'import.invalid' && verificationLink) {
|
|
18
|
+
try {
|
|
19
|
+
importedWechatChallenge = new URL(verificationLink.getAttribute('href') || '', source).hostname === 'mp.weixin.qq.com';
|
|
20
|
+
}
|
|
21
|
+
catch { /* Not a recognizable verification link. */ }
|
|
22
|
+
}
|
|
23
|
+
if (host === 'mp.weixin.qq.com' || importedWechatChallenge) {
|
|
24
|
+
if (/^\/mp\/wappoc_appmsgcaptcha(?:\/|$)/i.test(url.pathname))
|
|
25
|
+
throw new PageAccessError('微信');
|
|
26
|
+
const content = document.querySelector('#js_content')?.textContent?.trim() || '';
|
|
27
|
+
const articleText = document.querySelector('article')?.textContent?.trim() || '';
|
|
28
|
+
if (content.length >= 40 || articleText.length >= 300)
|
|
29
|
+
return;
|
|
30
|
+
const environmentNotice = /当前环境异常/.test(text) && /完成验证|去验证/.test(text);
|
|
31
|
+
if (text.length < 2000 && environmentNotice && (verificationLink || title === '环境异常' || heading === '环境异常'))
|
|
32
|
+
throw new PageAccessError('微信');
|
|
33
|
+
}
|
|
34
|
+
const challengeTitle = /^(?:just a moment[.!…]*|attention required!?\s*\|\s*cloudflare)$/i.test(title);
|
|
35
|
+
const challengeForm = document.querySelector('#challenge-form,#challenge-running,form[action*="/cdn-cgi/challenge-platform/"]');
|
|
36
|
+
if (text.length < 3000 && challengeTitle && challengeForm)
|
|
37
|
+
throw new PageAccessError('网站');
|
|
38
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export interface PurgeTicket {
|
|
2
|
+
id: string;
|
|
3
|
+
source: string;
|
|
4
|
+
staged: string;
|
|
5
|
+
manifest: string;
|
|
6
|
+
}
|
|
7
|
+
/** Durable, confirmed cleanup work. Only owned material directories are removable. */
|
|
8
|
+
export declare class PurgeStore {
|
|
9
|
+
private root;
|
|
10
|
+
readonly directory: string;
|
|
11
|
+
constructor(root: string);
|
|
12
|
+
private contained;
|
|
13
|
+
private tree;
|
|
14
|
+
private ticket;
|
|
15
|
+
validate(id: string, source: string): void;
|
|
16
|
+
list(): PurgeTicket[];
|
|
17
|
+
stage(id: string, source: string): PurgeTicket;
|
|
18
|
+
rollback(ticket: PurgeTicket): void;
|
|
19
|
+
finish(ticket: PurgeTicket): void;
|
|
20
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
3
|
+
const validId = (id) => typeof id === 'string' && /^r_[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(id);
|
|
4
|
+
/** Durable, confirmed cleanup work. Only owned material directories are removable. */
|
|
5
|
+
export class PurgeStore {
|
|
6
|
+
root;
|
|
7
|
+
directory;
|
|
8
|
+
constructor(root) {
|
|
9
|
+
this.root = root;
|
|
10
|
+
this.directory = join(root, '.system', 'purge');
|
|
11
|
+
}
|
|
12
|
+
contained(path) {
|
|
13
|
+
const rel = relative(this.root, resolve(path));
|
|
14
|
+
if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel))
|
|
15
|
+
throw new Error('清理路径超出资料库');
|
|
16
|
+
let current = this.root;
|
|
17
|
+
if (lstatSync(current).isSymbolicLink())
|
|
18
|
+
throw new Error('清理不支持符号链接');
|
|
19
|
+
for (const part of rel.split(sep)) {
|
|
20
|
+
current = join(current, part);
|
|
21
|
+
try {
|
|
22
|
+
if (lstatSync(current).isSymbolicLink())
|
|
23
|
+
throw new Error('清理不支持符号链接');
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (error.code !== 'ENOENT')
|
|
27
|
+
throw error;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
tree(path) {
|
|
32
|
+
this.contained(path);
|
|
33
|
+
const info = lstatSync(path);
|
|
34
|
+
if (info.isSymbolicLink())
|
|
35
|
+
throw new Error('清理不支持符号链接');
|
|
36
|
+
if (info.isDirectory())
|
|
37
|
+
for (const entry of readdirSync(path))
|
|
38
|
+
this.tree(join(path, entry));
|
|
39
|
+
}
|
|
40
|
+
ticket(id, source) {
|
|
41
|
+
if (!validId(id))
|
|
42
|
+
throw new Error('清理资料 ID 无效');
|
|
43
|
+
source = resolve(source);
|
|
44
|
+
const rel = relative(join(this.root, 'materials'), source);
|
|
45
|
+
if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel) || basename(source) !== id)
|
|
46
|
+
throw new Error('清理目标不是资料目录');
|
|
47
|
+
const ticket = { id, source, staged: join(this.directory, id), manifest: join(this.directory, `${id}.json`) };
|
|
48
|
+
for (const path of [ticket.source, ticket.staged, ticket.manifest])
|
|
49
|
+
this.contained(path);
|
|
50
|
+
return ticket;
|
|
51
|
+
}
|
|
52
|
+
validate(id, source) { this.tree(this.ticket(id, source).source); }
|
|
53
|
+
list() {
|
|
54
|
+
this.contained(this.directory);
|
|
55
|
+
if (!existsSync(this.directory))
|
|
56
|
+
return [];
|
|
57
|
+
return readdirSync(this.directory).filter(name => name.endsWith('.json')).sort().map(name => {
|
|
58
|
+
const manifest = join(this.directory, name);
|
|
59
|
+
this.contained(manifest);
|
|
60
|
+
const data = JSON.parse(readFileSync(manifest, 'utf8'));
|
|
61
|
+
if (data.schemaVersion !== 1 || !validId(data.id) || name !== `${data.id}.json` || typeof data.source !== 'string' || isAbsolute(data.source))
|
|
62
|
+
throw new Error('待清理记录损坏');
|
|
63
|
+
return this.ticket(data.id, resolve(this.root, data.source));
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
stage(id, source) {
|
|
67
|
+
const ticket = this.ticket(id, source);
|
|
68
|
+
this.tree(ticket.source);
|
|
69
|
+
if (existsSync(ticket.staged))
|
|
70
|
+
throw new Error('该资料仍有待完成的文件清理');
|
|
71
|
+
mkdirSync(this.directory, { recursive: true });
|
|
72
|
+
writeFileSync(ticket.manifest, JSON.stringify({ schemaVersion: 1, id, source: relative(this.root, source) }), { flag: 'wx' });
|
|
73
|
+
try {
|
|
74
|
+
renameSync(ticket.source, ticket.staged);
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
unlinkSync(ticket.manifest);
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
return ticket;
|
|
81
|
+
}
|
|
82
|
+
rollback(ticket) {
|
|
83
|
+
const checked = this.ticket(ticket.id, ticket.source);
|
|
84
|
+
if (existsSync(checked.source))
|
|
85
|
+
throw new Error('原目录已有内容,无法回退文件清理');
|
|
86
|
+
this.tree(checked.staged);
|
|
87
|
+
renameSync(checked.staged, checked.source);
|
|
88
|
+
unlinkSync(checked.manifest);
|
|
89
|
+
}
|
|
90
|
+
finish(ticket) {
|
|
91
|
+
const checked = this.ticket(ticket.id, ticket.source);
|
|
92
|
+
if (existsSync(checked.source)) {
|
|
93
|
+
if (existsSync(checked.staged))
|
|
94
|
+
throw new Error('清理目录与原目录冲突,文件已保留');
|
|
95
|
+
// The process stopped before the move; no original file may be deleted.
|
|
96
|
+
unlinkSync(checked.manifest);
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (existsSync(checked.staged)) {
|
|
100
|
+
this.tree(checked.staged);
|
|
101
|
+
// Both the absolute boundary and every descendant were checked above.
|
|
102
|
+
rmSync(checked.staged, { recursive: true, maxRetries: 2, retryDelay: 50 });
|
|
103
|
+
}
|
|
104
|
+
unlinkSync(checked.manifest);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import type { DirectoryListing } from '@deepseek-ai/dsh-host-directory-picker';
|
|
3
|
+
import { Library } from './core/library.js';
|
|
4
|
+
import type { FetchPolicy } from './core/fetch.js';
|
|
5
|
+
export type { DirectoryListing };
|
|
6
|
+
export interface DirectorySettings {
|
|
7
|
+
status(): {
|
|
8
|
+
canChange: boolean;
|
|
9
|
+
canChangeNetwork: boolean;
|
|
10
|
+
picker: 'native' | 'browse' | null;
|
|
11
|
+
allowProxyFakeIp: boolean;
|
|
12
|
+
};
|
|
13
|
+
pick(signal: AbortSignal): Promise<string | null>;
|
|
14
|
+
list(path: string | undefined, signal: AbortSignal): Promise<DirectoryListing>;
|
|
15
|
+
change(root: string, expectedRoot: string): Promise<string>;
|
|
16
|
+
setProxyMode(enabled: boolean, expected: boolean, expectedRoot: string): Promise<boolean>;
|
|
17
|
+
}
|
|
18
|
+
export declare function createDirectorySettings(ctx: Context, library: Library, policy?: FetchPolicy): DirectorySettings;
|