dsh-knowledge-library 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/README.md +123 -18
  3. package/dist/client.js +1699 -374
  4. package/dist/core/extract.d.ts +1 -0
  5. package/dist/core/extract.js +62 -5
  6. package/dist/core/fetch.js +82 -11
  7. package/dist/core/grouping.d.ts +6 -0
  8. package/dist/core/grouping.js +71 -0
  9. package/dist/core/incremental.d.ts +14 -0
  10. package/dist/core/incremental.js +52 -0
  11. package/dist/core/jobs.d.ts +1 -0
  12. package/dist/core/jobs.js +4 -0
  13. package/dist/core/knowledge.d.ts +37 -0
  14. package/dist/core/knowledge.js +617 -0
  15. package/dist/core/library.d.ts +20 -2
  16. package/dist/core/library.js +271 -12
  17. package/dist/core/page-access.d.ts +6 -0
  18. package/dist/core/page-access.js +38 -0
  19. package/dist/core/purge.d.ts +20 -0
  20. package/dist/core/purge.js +106 -0
  21. package/dist/directory-settings.d.ts +18 -0
  22. package/dist/directory-settings.js +92 -0
  23. package/dist/http.d.ts +3 -1
  24. package/dist/http.js +121 -4
  25. package/dist/index.js +17 -5
  26. package/dist/knowledge-model.d.ts +18 -0
  27. package/dist/knowledge-model.js +116 -0
  28. package/dist/preview-settings.d.ts +6 -0
  29. package/dist/preview-settings.js +42 -0
  30. package/dist/public/app.css +385 -182
  31. package/dist/public/app.js +11 -7
  32. package/dist/public/index.html +1 -1
  33. package/dist/standalone.js +8 -3
  34. package/dist/tools.d.ts +2 -1
  35. package/dist/tools.js +10 -1
  36. package/dist/types.d.ts +143 -1
  37. package/dist/ui/App.js +244 -47
  38. package/dist/ui/Dialog.d.ts +7 -0
  39. package/dist/ui/Dialog.js +32 -0
  40. package/dist/ui/Grouping.d.ts +12 -0
  41. package/dist/ui/Grouping.js +93 -0
  42. package/dist/ui/Knowledge.d.ts +26 -0
  43. package/dist/ui/Knowledge.js +106 -0
  44. package/dist/ui/api.d.ts +7 -0
  45. package/dist/ui/api.js +16 -0
  46. package/dist/ui/icons.js +2 -0
  47. package/package.json +4 -1
@@ -2,6 +2,7 @@ import type { Segment } from '../types.js';
2
2
  export declare function extractArticle(html: string, url: string): {
3
3
  title: string;
4
4
  author: string | undefined;
5
+ extraction: string;
5
6
  content: string;
6
7
  };
7
8
  export declare function parseSubtitles(input: string): Segment[];
@@ -1,6 +1,7 @@
1
1
  import { parseHTML } from 'linkedom';
2
2
  import { Readability } from '@mozilla/readability';
3
3
  import TurndownService from 'turndown';
4
+ import { assertArticleAccess } from './page-access.js';
4
5
  const markdown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
5
6
  markdown.remove(['script', 'style', 'iframe', 'form', 'input', 'button']);
6
7
  function parseDocument(html, url) {
@@ -16,20 +17,70 @@ function parseDocument(html, url) {
16
17
  Object.defineProperties(document, { documentURI: { value: url }, baseURI: { value: baseURI } });
17
18
  return document;
18
19
  }
20
+ function extractWechatArticle(document, url) {
21
+ const host = new URL(url).hostname.toLowerCase().replace(/\.$/, '');
22
+ const body = document.querySelector('#js_content');
23
+ // Saved WeChat HTML has no HTTP source when imported as a local file.
24
+ const savedWechat = host === 'import.invalid' && body?.classList.contains('rich_media_content')
25
+ && document.querySelector('#activity-name.rich_media_title');
26
+ if (host !== 'mp.weixin.qq.com' && !savedWechat)
27
+ return null;
28
+ if (!body || (body.textContent?.trim().length ?? 0) < 40)
29
+ return null;
30
+ // WeChat hides the entire body until its scripts run. Read the explicit article
31
+ // container without running those scripts or discarding it as invisible text.
32
+ return {
33
+ title: document.querySelector('#activity-name')?.textContent?.trim()
34
+ || document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim()
35
+ || document.title,
36
+ byline: document.querySelector('#js_author_name')?.textContent?.trim()
37
+ || document.querySelector('#js_name')?.textContent?.trim()
38
+ || document.querySelector('meta[name="author"]')?.getAttribute('content')?.trim(),
39
+ content: body.innerHTML,
40
+ textContent: body.textContent,
41
+ };
42
+ }
43
+ function normalizeWechatCode(document) {
44
+ for (const pre of Array.from(document.querySelectorAll('pre'))) {
45
+ const lines = Array.from(pre.children).filter(node => node.localName === 'code');
46
+ if (!lines.length || lines.length !== pre.children.length)
47
+ continue;
48
+ // WeChat uses one sibling <code> element per line; Turndown's standard
49
+ // fenced-code rule only reads the first <code>. Merge before conversion.
50
+ const code = document.createElement('code');
51
+ code.className = lines[0].className;
52
+ code.textContent = lines.map(line => {
53
+ for (const br of Array.from(line.querySelectorAll('br')))
54
+ br.replaceWith('\n');
55
+ return (line.textContent || '').replace(/\u00a0/g, ' ').replace(/\n$/, '');
56
+ }).join('\n');
57
+ const language = pre.getAttribute('data-lang');
58
+ if (language && /^[\w+-]+$/.test(language))
59
+ code.className = `language-${language}`;
60
+ pre.replaceChildren(code);
61
+ }
62
+ }
19
63
  export function extractArticle(html, url) {
20
64
  // Linkedom parses inert DOM only: no script execution, windows, or resource loading.
21
65
  // It also avoids the punycode/ dependency that DSH 0.2.1-rc.2 misroutes in packed profiles.
22
66
  const document = parseDocument(html, url);
23
- for (const node of Array.from(document.querySelectorAll('script,style,iframe,form')))
67
+ for (const node of Array.from(document.querySelectorAll('script,style,iframe')))
24
68
  node.remove();
25
- const article = new Readability(document).parse();
69
+ assertArticleAccess(document, url);
70
+ for (const node of Array.from(document.querySelectorAll('form')))
71
+ node.remove();
72
+ const wechat = extractWechatArticle(document, url);
73
+ const article = wechat || new Readability(document).parse();
26
74
  if (!article?.content || (article.textContent?.trim().length ?? 0) < 40) {
27
75
  throw new Error('未提取到有效正文。该网页可能需要登录或动态渲染,请复制正文或导入保存的 HTML。');
28
76
  }
29
- const clean = parseDocument(`<!doctype html><html><head></head><body>${article.content}</body></html>`, url);
77
+ const baseUrl = wechat ? document.baseURI : url;
78
+ const clean = parseDocument(`<!doctype html><html><head></head><body>${article.content}</body></html>`, baseUrl);
79
+ if (wechat)
80
+ normalizeWechatCode(clean);
30
81
  for (const anchor of Array.from(clean.querySelectorAll('a'))) {
31
82
  try {
32
- const href = new URL(anchor.getAttribute('href') ?? '', url);
83
+ const href = new URL(anchor.getAttribute('href') ?? '', baseUrl);
33
84
  if (['http:', 'https:'].includes(href.protocol))
34
85
  anchor.setAttribute('href', href.href);
35
86
  else
@@ -42,7 +93,12 @@ export function extractArticle(html, url) {
42
93
  // Preserve image references; do not silently fetch third-party resources.
43
94
  for (const image of Array.from(clean.querySelectorAll('img'))) {
44
95
  try {
45
- const src = new URL(image.getAttribute('src') ?? '', url);
96
+ const value = (wechat && image.getAttribute('data-src')) || image.getAttribute('src');
97
+ if (!value?.trim()) {
98
+ image.remove();
99
+ continue;
100
+ }
101
+ const src = new URL(value, baseUrl);
46
102
  if (['http:', 'https:'].includes(src.protocol))
47
103
  image.setAttribute('src', src.href);
48
104
  else
@@ -55,6 +111,7 @@ export function extractArticle(html, url) {
55
111
  return {
56
112
  title: article.title?.trim() || new URL(url).hostname,
57
113
  author: article.byline || undefined,
114
+ extraction: wechat ? '微信正文容器提取' : 'Readability 正文提取',
58
115
  content: markdown.turndown(clean.body.innerHTML).trim(),
59
116
  };
60
117
  }
@@ -2,7 +2,45 @@ import { request as httpRequest } from 'node:http';
2
2
  import { request as httpsRequest } from 'node:https';
3
3
  import { lookup } from 'node:dns/promises';
4
4
  import { isIP } from 'node:net';
5
+ import { setTimeout as delay } from 'node:timers/promises';
5
6
  const MAX_BYTES = 6 * 1024 * 1024;
7
+ const MAX_RETRIES = 2;
8
+ const TRANSIENT_ERRORS = new Set(['ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'ETIMEDOUT', 'EAI_AGAIN', 'ENETUNREACH', 'EHOSTUNREACH', 'ERR_STREAM_PREMATURE_CLOSE']);
9
+ function errorCode(error) {
10
+ return error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
11
+ }
12
+ function networkError(error, retries) {
13
+ const code = errorCode(error);
14
+ if (TRANSIENT_ERRORS.has(code)) {
15
+ const reason = code === 'EAI_AGAIN' ? '网页域名暂时无法解析' : '网页连接中断或无法建立';
16
+ return new Error(`${reason},已自动重试 ${retries} 次仍未成功。请检查网络或代理后重新采集,也可以打开原网页复制正文。`, { cause: error });
17
+ }
18
+ if (code === 'ENOTFOUND')
19
+ return new Error('网页域名解析失败,请检查网址、网络或代理设置。', { cause: error });
20
+ if (/CERT|SELF_SIGNED|UNABLE_TO_VERIFY_LEAF_SIGNATURE/.test(code)) {
21
+ return new Error('网页 HTTPS 证书校验失败,请检查系统时间、网络代理或站点证书后重试。', { cause: error });
22
+ }
23
+ if (code === 'EPROTO' || /^ERR_(?:TLS|SSL)_/.test(code)) {
24
+ return new Error('无法与网页建立安全的 HTTPS 连接,请检查网络、代理或站点的 TLS 配置。', { cause: error });
25
+ }
26
+ return error;
27
+ }
28
+ function abortable(operation, signal) {
29
+ return new Promise((resolve, reject) => {
30
+ const onAbort = () => reject(signal.reason);
31
+ if (signal.aborted)
32
+ reject(signal.reason);
33
+ else
34
+ signal.addEventListener('abort', onAbort, { once: true });
35
+ operation.then(value => {
36
+ signal.removeEventListener('abort', onAbort);
37
+ resolve(value);
38
+ }, error => {
39
+ signal.removeEventListener('abort', onAbort);
40
+ reject(error);
41
+ });
42
+ });
43
+ }
6
44
  export function isPublicAddress(address) {
7
45
  if (isIP(address) === 4) {
8
46
  const [a, b, c] = address.split('.').map(Number);
@@ -39,28 +77,28 @@ export function permittedAddress(hostname, address, policy) {
39
77
  if (isPublicAddress(address))
40
78
  return true;
41
79
  // An explicit deployment opt-in for Clash/Mihomo TUN fake DNS, not a general private-network exemption.
42
- return policy.allowProxyFakeIp === true && isIP(hostname) === 0
43
- && hostname.includes('.') && !/\.(localhost|local|internal|lan)$/i.test(hostname)
80
+ const normalized = hostname.replace(/\.$/, '');
81
+ return policy.allowProxyFakeIp === true && isIP(address) === 4 && isIP(normalized) === 0
82
+ && normalized.includes('.') && !/\.(localhost|local|internal|lan)$/i.test(normalized)
44
83
  && /^198\.(18|19)\.\d{1,3}\.\d{1,3}$/.test(address);
45
84
  }
46
- export async function fetchPublicPage(input, signal, policy = {}) {
85
+ async function fetchPageOnce(input, signal, policy) {
47
86
  let url = validateUrl(input);
48
- const timeout = AbortSignal.timeout(25_000);
49
- const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
50
87
  for (let redirect = 0; redirect <= 4; redirect++) {
51
- combined.throwIfAborted();
88
+ signal.throwIfAborted();
52
89
  const hostname = url.hostname.replace(/^\[|\]$/g, '');
53
- const addresses = await lookup(hostname, { all: true });
90
+ const addresses = await abortable(lookup(hostname, { all: true }), signal);
91
+ signal.throwIfAborted();
54
92
  if (!addresses.length || addresses.some(item => !permittedAddress(hostname, item.address, policy))) {
55
93
  if (addresses.some(item => /^198\.(18|19)\./.test(item.address)))
56
- throw new Error('检测到代理 fake-IP DNS。若本机使用 Clash/Mihomo TUN,请启用插件的 allowProxyFakeIp;否则请直接导入文本。');
94
+ throw new Error('检测到代理 fake-IP DNS。若本机使用 Clash/Mihomo TUN,请在资料库设置中启用“代理兼容”,然后重新采集;否则请复制正文导入。');
57
95
  throw new Error('网页采集不访问本机、内网或保留地址,请直接导入文件或文本');
58
96
  }
59
97
  const address = addresses[0];
60
98
  const response = await new Promise((resolve, reject) => {
61
99
  const req = (url.protocol === 'https:' ? httpsRequest : httpRequest)(url, {
62
- method: 'GET', signal: combined, family: address.family,
63
- headers: { 'User-Agent': 'DSH-Knowledge-Library/0.1', Accept: 'text/html,text/plain;q=0.8', 'Accept-Encoding': 'identity' },
100
+ method: 'GET', signal, family: address.family, agent: false,
101
+ headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DSH-Knowledge-Library/0.1)', Accept: 'text/html,application/xhtml+xml,text/plain;q=0.8', 'Accept-Encoding': 'identity' },
64
102
  // Pin the validated DNS result so a second resolution cannot reach an internal host.
65
103
  lookup: (_host, _options, callback) => callback(null, address.address, address.family),
66
104
  }, res => {
@@ -69,12 +107,15 @@ export async function fetchPublicPage(input, signal, policy = {}) {
69
107
  res.on('data', (chunk) => {
70
108
  size += chunk.length;
71
109
  if (size > MAX_BYTES) {
72
- req.destroy(new Error('网页超过 6 MB,请下载后手动导入'));
110
+ const error = new Error('网页超过 6 MB,请下载后手动导入');
111
+ reject(error);
112
+ req.destroy(error);
73
113
  return;
74
114
  }
75
115
  chunks.push(chunk);
76
116
  });
77
117
  res.on('end', () => resolve({ status: res.statusCode ?? 0, location: res.headers.location, contentType: String(res.headers['content-type'] ?? ''), bytes: Buffer.concat(chunks) }));
118
+ res.on('aborted', () => reject(Object.assign(new Error('网页响应在传输完成前断开'), { code: 'ECONNRESET' })));
78
119
  res.on('error', reject);
79
120
  });
80
121
  req.on('error', reject);
@@ -103,3 +144,33 @@ export async function fetchPublicPage(input, signal, policy = {}) {
103
144
  }
104
145
  throw new Error('网页重定向次数过多');
105
146
  }
147
+ export async function fetchPublicPage(input, signal, policy = {}) {
148
+ const timeout = AbortSignal.timeout(25_000);
149
+ const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
150
+ const snapshot = { allowProxyFakeIp: policy.allowProxyFakeIp === true };
151
+ let retries = 0;
152
+ try {
153
+ for (;;) {
154
+ combined.throwIfAborted();
155
+ try {
156
+ return await fetchPageOnce(input, combined, snapshot);
157
+ }
158
+ catch (error) {
159
+ combined.throwIfAborted();
160
+ if (!TRANSIENT_ERRORS.has(errorCode(error)) || retries >= MAX_RETRIES)
161
+ throw error;
162
+ // Retry a fresh GET from the original URL, rechecking every DNS result and
163
+ // redirect. DNS, requests and backoff all share the same 25-second limit.
164
+ await delay(350 * 2 ** retries + Math.floor(Math.random() * 150), undefined, { signal: combined });
165
+ retries++;
166
+ }
167
+ }
168
+ }
169
+ catch (error) {
170
+ if (signal?.aborted)
171
+ throw signal.reason;
172
+ if (timeout.aborted)
173
+ throw new Error('网页连接超时,请检查网络或代理后重试,也可以复制正文导入。', { cause: error });
174
+ throw networkError(error, retries);
175
+ }
176
+ }
@@ -0,0 +1,6 @@
1
+ import type { GroupingPlan, MaterialDetail, SourceAnalysis } from '../types.js';
2
+ export declare function groupingTags(value: unknown): string[];
3
+ export declare function parseModelJson(output: string): unknown;
4
+ export declare function sourceAnalysis(value: unknown, source: MaterialDetail): SourceAnalysis;
5
+ /** A plan is a complete partition: every selected source occurs exactly once. */
6
+ export declare function groupingPlan(value: unknown, sourceIds: string[], analyses: SourceAnalysis[], fromModel?: boolean): GroupingPlan;
@@ -0,0 +1,71 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { categoryName } from './library.js';
3
+ const text = (value, max, label, empty = false) => {
4
+ if (typeof value !== 'string' || value.length > max || (!empty && !value.trim()))
5
+ throw new Error(`${label}格式不正确`);
6
+ return value.trim();
7
+ };
8
+ export function groupingTags(value) {
9
+ if (!Array.isArray(value) || value.length > 12)
10
+ throw new Error('建议标签最多12个');
11
+ return [...new Set(value.map(v => text(v, 40, '标签')).filter(Boolean))];
12
+ }
13
+ export function parseModelJson(output) {
14
+ try {
15
+ return JSON.parse(output.trim().replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```$/, ''));
16
+ }
17
+ catch {
18
+ throw new Error('模型没有返回有效的整理格式,请重试');
19
+ }
20
+ }
21
+ export function sourceAnalysis(value, source) {
22
+ const data = value;
23
+ if (!data || data.sourceId !== source.id || !['clear', 'uncertain'].includes(data.certainty))
24
+ throw new Error('逐篇分析的来源或确定性无效');
25
+ if (!Array.isArray(data.citations) || !data.citations.length || data.citations.length > 20 || data.citations.some(c => c?.materialId !== source.id || !source.segments.some(s => s.id === c.segmentId)))
26
+ throw new Error('逐篇分析包含无效来源引用');
27
+ const concepts = groupingTags(data.concepts);
28
+ const summary = text(data.summary, 2500, '资料概要');
29
+ if (/library:\/\//.test(summary))
30
+ throw new Error('逐篇概要请使用独立引用列表,不在概要编写链接');
31
+ return { sourceId: source.id, theme: text(data.theme, 200, '主题'), summary, concepts, usage: text(data.usage, 800, '用途'), category: categoryName(text(data.category, 180, '建议分类')), tags: groupingTags(data.tags), certainty: data.certainty, uncertainty: text(data.uncertainty, 800, '待核实说明', data.certainty === 'clear'), citations: data.citations.map(c => ({ materialId: c.materialId, segmentId: c.segmentId })) };
32
+ }
33
+ /** A plan is a complete partition: every selected source occurs exactly once. */
34
+ export function groupingPlan(value, sourceIds, analyses, fromModel = false) {
35
+ const data = value;
36
+ if (!data || !Array.isArray(data.groups) || !Array.isArray(data.unassigned) || data.groups.length > sourceIds.length || data.unassigned.length > sourceIds.length)
37
+ throw new Error('分组方案格式不正确');
38
+ const seen = new Set();
39
+ const groupIds = new Set();
40
+ const accept = (id) => {
41
+ if (typeof id !== 'string' || !sourceIds.includes(id) || seen.has(id))
42
+ throw new Error('分组资料重复、遗漏或超出所选范围');
43
+ seen.add(id);
44
+ return id;
45
+ };
46
+ const groups = data.groups.map(group => {
47
+ if (!group || !Array.isArray(group.sourceIds) || !group.sourceIds.length)
48
+ throw new Error('每个分组必须包含资料');
49
+ const id = fromModel ? `group_${randomUUID()}` : group.id;
50
+ if (typeof id !== 'string' || !/^group_[a-f0-9-]{36}$/.test(id) || groupIds.has(id))
51
+ throw new Error('分组ID无效或重复');
52
+ groupIds.add(id);
53
+ return { id, title: text(group.title, 200, '分组名称'), reason: text(group.reason, 1200, '分组理由'), sourceIds: group.sourceIds.map(accept), category: categoryName(text(group.category, 180, '来源分类')), tags: groupingTags(group.tags) };
54
+ });
55
+ const unassigned = data.unassigned.map(item => { if (!item)
56
+ throw new Error('待整理项无效'); return { sourceId: accept(item.sourceId), reason: text(item.reason, 1200, '待整理理由') }; });
57
+ if (seen.size !== sourceIds.length)
58
+ throw new Error('分组资料重复、遗漏或超出所选范围');
59
+ if (fromModel) {
60
+ // Uncertain evidence stays pending until a human deliberately assigns it.
61
+ for (const analysis of analyses.filter(a => a.certainty === 'uncertain')) {
62
+ for (const group of groups)
63
+ group.sourceIds = group.sourceIds.filter(id => id !== analysis.sourceId);
64
+ if (!unassigned.some(item => item.sourceId === analysis.sourceId))
65
+ unassigned.push({ sourceId: analysis.sourceId, reason: analysis.uncertainty || '资料信息不足,需进一步核实' });
66
+ }
67
+ }
68
+ if (data.applyClassification !== undefined && typeof data.applyClassification !== 'boolean')
69
+ throw new Error('同步分类选项无效');
70
+ return { groups: groups.filter(group => group.sourceIds.length), unassigned, applyClassification: fromModel ? true : data.applyClassification ?? true };
71
+ }
@@ -0,0 +1,14 @@
1
+ import type { Library } from './library.js';
2
+ import type { KnowledgeDraft, MaterialDetail, NoteIntegration } from '../types.js';
3
+ export declare const noteFingerprint: (note: MaterialDetail) => string;
4
+ /** Local lexical recall, followed by full-body model comparison; no silent body truncation. */
5
+ export declare function recallNotes(library: Library, draft: KnowledgeDraft, topic: string, sources: MaterialDetail[]): {
6
+ candidates: MaterialDetail[];
7
+ search: {
8
+ scanned: number;
9
+ matched: number;
10
+ compared: number;
11
+ oversized: number;
12
+ };
13
+ };
14
+ export declare function integrationValue(value: unknown, candidates: MaterialDetail[], search: NoteIntegration['search']): NoteIntegration;
@@ -0,0 +1,52 @@
1
+ import { createHash } from 'node:crypto';
2
+ export const noteFingerprint = (note) => createHash('sha256').update(JSON.stringify([note.content, note.segments, note.title, note.category, note.tags, note.citations, note.revision, note.updatedAt, note.deletedAt, note.deletionId])).digest('hex');
3
+ const tokens = (text) => [...new Set((text.toLowerCase().match(/[a-z0-9_]{2,}|[\p{Script=Han}]+/gu) || []).flatMap(term => /^[\p{Script=Han}]/u.test(term) ? Array.from({ length: Math.max(0, term.length - 1) }, (_, index) => term.slice(index, index + 2)) : [term]))].slice(0, 500);
4
+ /** Local lexical recall, followed by full-body model comparison; no silent body truncation. */
5
+ export function recallNotes(library, draft, topic, sources) {
6
+ const rows = library.db.prepare('SELECT id,meta,content FROM materials ORDER BY updated_at DESC').all();
7
+ const query = tokens([topic, draft.title, ...draft.tags, ...sources.flatMap(s => [s.title, ...s.tags]), draft.content].join(' '));
8
+ const notes = rows.map(row => ({ ...row, metadata: JSON.parse(row.meta) })).filter(row => row.metadata.kind === 'note' && !row.metadata.deletedAt);
9
+ const ranked = notes.map(row => {
10
+ const title = `${row.metadata.title} ${row.metadata.tags.join(' ')}`.toLowerCase();
11
+ const body = row.content.toLowerCase();
12
+ return { ...row, score: query.reduce((sum, term) => sum + (title.includes(term) ? 4 : body.includes(term) ? 1 : 0), 0) };
13
+ }).filter(row => row.score > 0).sort((a, b) => b.score - a.score);
14
+ let size = 0;
15
+ let oversized = 0;
16
+ const candidates = [];
17
+ for (const row of ranked) {
18
+ if (row.content.length > 50_000) {
19
+ oversized++;
20
+ continue;
21
+ }
22
+ if (candidates.length >= 5 || size + row.content.length > 60_000)
23
+ continue;
24
+ const note = library.get(row.id);
25
+ candidates.push(note);
26
+ size += note.content.length;
27
+ }
28
+ return { candidates, search: { scanned: notes.length, matched: ranked.length, compared: candidates.length, oversized } };
29
+ }
30
+ export function integrationValue(value, candidates, search) {
31
+ const data = value;
32
+ const bounded = (v, max, label, empty = false) => {
33
+ if (typeof v !== 'string' || v.length > max || (!empty && !v.trim()))
34
+ throw new Error(`增量整理的${label}无效`);
35
+ return v.trim();
36
+ };
37
+ const list = (v) => { if (!Array.isArray(v) || v.length > 12)
38
+ throw new Error('增量对比列表格式无效'); return v.map(item => bounded(item, 1000, '对比说明')); };
39
+ if (!data || !['new', 'append', 'duplicate'].includes(data.action))
40
+ throw new Error('增量整理建议格式无效');
41
+ const recommendation = data.action;
42
+ const target = recommendation === 'new' ? undefined : candidates.find(n => n.id === data.noteId);
43
+ if (recommendation !== 'new' && !target)
44
+ throw new Error('增量整理指定了未检索的笔记');
45
+ const content = bounded(data.content, 50_000, '新增正文', recommendation !== 'append');
46
+ if (recommendation !== 'append' && content)
47
+ throw new Error('新建或重复建议不应包含追加正文');
48
+ if (target && content && target.content.includes(content))
49
+ throw new Error('建议追加的正文已存在,请重新核对');
50
+ return { recommendation, choice: recommendation, reason: bounded(data.reason, 2000, '建议理由'), overlaps: list(data.overlaps), additions: list(data.additions), conflicts: list(data.conflicts), content, search,
51
+ ...(target ? { target: { id: target.id, title: target.title, category: target.category, revision: target.revision, fingerprint: noteFingerprint(target), content: target.content } } : {}) };
52
+ }
@@ -20,6 +20,7 @@ export declare class ImportQueue {
20
20
  attempts: number;
21
21
  materialId?: string;
22
22
  error?: string;
23
+ errorCode?: "PAGE_REQUIRES_BROWSER";
23
24
  }[];
24
25
  add(url: string, fields?: CaptureFields): ImportJob;
25
26
  retry(id: string): ImportJob;
package/dist/core/jobs.js CHANGED
@@ -3,6 +3,7 @@ import { join } from 'node:path';
3
3
  import { randomUUID } from 'node:crypto';
4
4
  import { atomicJson } from './library.js';
5
5
  import { fetchPublicPage, validateUrl } from './fetch.js';
6
+ import { PageAccessError } from './page-access.js';
6
7
  export class ImportQueue {
7
8
  library;
8
9
  fetchPage;
@@ -48,6 +49,7 @@ export class ImportQueue {
48
49
  throw new Error('只能重试失败的任务');
49
50
  job.status = 'queued';
50
51
  job.error = undefined;
52
+ job.errorCode = undefined;
51
53
  job.attempts = 0;
52
54
  this.persist();
53
55
  this.kick();
@@ -75,10 +77,12 @@ export class ImportQueue {
75
77
  job.materialId = result.material.id;
76
78
  job.status = 'completed';
77
79
  job.error = undefined;
80
+ job.errorCode = undefined;
78
81
  }
79
82
  catch (error) {
80
83
  job.status = this.stopped ? 'queued' : 'failed';
81
84
  job.error = error.message;
85
+ job.errorCode = error instanceof PageAccessError ? error.code : undefined;
82
86
  }
83
87
  job.updatedAt = new Date().toISOString();
84
88
  this.persist();
@@ -0,0 +1,37 @@
1
+ import { Library } from './library.js';
2
+ import type { KnowledgeJob, KnowledgeOverview } from '../types.js';
3
+ import type { KnowledgeModel } from '../knowledge-model.js';
4
+ export declare class KnowledgeQueue {
5
+ private library;
6
+ private model;
7
+ private jobs;
8
+ private file;
9
+ private stopped;
10
+ private running?;
11
+ private abort?;
12
+ constructor(library: Library, model: KnowledgeModel);
13
+ overview(): KnowledgeOverview;
14
+ get(id: string): KnowledgeJob;
15
+ private find;
16
+ private persist;
17
+ private sources;
18
+ private checkedSources;
19
+ add(ids: unknown, topic?: unknown, requestedMode?: unknown): KnowledgeJob;
20
+ private newJob;
21
+ cancel(id: string): KnowledgeJob;
22
+ discard(id: string): KnowledgeJob;
23
+ editPlan(id: string, value: unknown, version: unknown): KnowledgeJob;
24
+ confirmPlan(id: string, version: unknown, apply?: unknown, expectedModel?: unknown): KnowledgeJob;
25
+ private runClassification;
26
+ rollbackPlan(id: string): KnowledgeJob;
27
+ edit(id: string, value: unknown, version: unknown, integration?: unknown): KnowledgeJob;
28
+ confirm(id: string, version: unknown, category?: unknown): import("../types.js").CaptureResult;
29
+ private progress;
30
+ private call;
31
+ private evidence;
32
+ private analyze;
33
+ private generate;
34
+ private kick;
35
+ idle(): Promise<void>;
36
+ close(): Promise<void>;
37
+ }