dsh-plugin-litsearch 0.2.0 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -38
- package/client/client.js +4750 -2342
- package/lib/host/export/bibtex.js +1 -4
- package/lib/host/export/common.d.ts +0 -2
- package/lib/host/export/common.js +0 -7
- package/lib/host/export/csv.js +1 -5
- package/lib/host/export/markdown.d.ts +1 -1
- package/lib/host/export/markdown.js +9 -26
- package/lib/host/export/ris.js +1 -4
- package/lib/host/paths.d.ts +4 -3
- package/lib/host/paths.js +4 -3
- package/lib/host/pdf/root.d.ts +11 -0
- package/lib/host/pdf/root.js +27 -0
- package/lib/host/pdf/store.d.ts +17 -7
- package/lib/host/pdf/store.js +110 -32
- package/lib/host/router.js +101 -87
- package/lib/host/tasks/filter.d.ts +3 -1
- package/lib/host/tasks/filter.js +45 -4
- package/lib/host/tasks/runner.js +1 -8
- package/lib/host/tasks/service.d.ts +21 -13
- package/lib/host/tasks/service.js +158 -47
- package/lib/host/tasks/states.d.ts +1 -3
- package/lib/host/tasks/states.js +2 -66
- package/lib/host/tasks/stats.js +9 -4
- package/lib/host/tasks/store.d.ts +1 -1
- package/lib/host/tasks/store.js +1 -1
- package/lib/host/tasks/summary.js +1 -1
- package/lib/host/tools.d.ts +1 -1
- package/lib/host/tools.js +25 -38
- package/lib/host/workspace.d.ts +20 -0
- package/lib/host/workspace.js +32 -0
- package/lib/index.d.ts +17 -8
- package/lib/index.js +33 -11
- package/lib/search/aggregate.js +1 -0
- package/lib/search/config.d.ts +17 -1
- package/lib/search/config.js +16 -1
- package/lib/search/identity.js +1 -0
- package/lib/search/oa.d.ts +8 -0
- package/lib/search/oa.js +53 -6
- package/lib/search/providers/googlescholar.d.ts +12 -0
- package/lib/search/providers/googlescholar.js +154 -0
- package/lib/search/providers/index.js +2 -0
- package/lib/search/query-adapter.js +3 -0
- package/lib/search/rate-limit.js +2 -1
- package/lib/search/types.d.ts +35 -13
- package/lib/search/types.js +25 -1
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* 模型工具与面板 HTTP 路由都只调用这里,保证两边操作同一份状态。
|
|
6
6
|
*/
|
|
7
7
|
import { randomUUID } from 'node:crypto';
|
|
8
|
-
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
8
|
+
import { mkdir, rm, stat, writeFile } from 'node:fs/promises';
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import { MAX_LIMIT, PLUGIN_VERSION, getSettings } from '../../search/config.js';
|
|
11
11
|
import { SearchError } from '../../search/errors.js';
|
|
@@ -17,8 +17,9 @@ import { fetchPubtatorEntities } from '../../search/pubtator-entities.js';
|
|
|
17
17
|
import { SOURCE_IDS } from '../../search/types.js';
|
|
18
18
|
import { TaskEvents } from '../events.js';
|
|
19
19
|
import { buildExport } from '../export/index.js';
|
|
20
|
-
import { cleanupPdfCache, epmcFullTextAvailable, persistPdf, probePdfRoot,
|
|
20
|
+
import { cleanupPdfCache, epmcFullTextAvailable, persistPdf, probePdfRoot, prunePdfCache, readSavedPdf, readableFilename, resolvePdf, resolvePdfRoot, } from '../pdf/store.js';
|
|
21
21
|
import { ZoteroConnector } from '../zotero/connector.js';
|
|
22
|
+
import { normalizeWorkspace } from '../workspace.js';
|
|
22
23
|
import { compareTasks } from './compare.js';
|
|
23
24
|
import { filterPapers } from './filter.js';
|
|
24
25
|
import { TaskRunner } from './runner.js';
|
|
@@ -31,6 +32,48 @@ const MAX_TITLE = 120;
|
|
|
31
32
|
const MAX_PAGE_LIMIT = 200;
|
|
32
33
|
const MAX_TYPES = 20;
|
|
33
34
|
const PROBE_QUERY = 'machine learning';
|
|
35
|
+
/** 已下载文件存在性检查的短缓存:详情轮询时不必反复 stat 同一批文件。 */
|
|
36
|
+
const PDF_EXISTS_TTL_MS = 10_000;
|
|
37
|
+
const pdfExistsCache = new Map();
|
|
38
|
+
async function pdfFileExists(filePath) {
|
|
39
|
+
const cached = pdfExistsCache.get(filePath);
|
|
40
|
+
if (cached !== undefined && Date.now() - cached.at < PDF_EXISTS_TTL_MS)
|
|
41
|
+
return cached.exists;
|
|
42
|
+
const exists = await stat(filePath)
|
|
43
|
+
.then((info) => info.isFile())
|
|
44
|
+
.catch(() => false);
|
|
45
|
+
pdfExistsCache.set(filePath, { exists, at: Date.now() });
|
|
46
|
+
return exists;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* 已下载的 PDF 可能被用户在文件管理器里删掉(也可能从废纸篓恢复):
|
|
50
|
+
* 每次读取任务时校准状态,删除后回到 `missing`,点击即可重新下载。
|
|
51
|
+
*/
|
|
52
|
+
async function reconcilePdfFiles(task) {
|
|
53
|
+
const downloads = Object.entries(task.states).filter(([, state]) => state.pdfFile !== undefined &&
|
|
54
|
+
(state.pdfStatus === 'downloaded' || state.pdfStatus === 'missing'));
|
|
55
|
+
if (downloads.length === 0)
|
|
56
|
+
return false;
|
|
57
|
+
const checks = await Promise.all(downloads.map(async ([paperId, state]) => ({
|
|
58
|
+
paperId,
|
|
59
|
+
exists: await pdfFileExists(state.pdfFile ?? ''),
|
|
60
|
+
})));
|
|
61
|
+
let changed = false;
|
|
62
|
+
for (const { paperId, exists } of checks) {
|
|
63
|
+
const current = task.states[paperId];
|
|
64
|
+
if (current === undefined)
|
|
65
|
+
continue;
|
|
66
|
+
if (current.pdfStatus === 'downloaded' && !exists) {
|
|
67
|
+
task.states[paperId] = { ...current, pdfStatus: 'missing' };
|
|
68
|
+
changed = true;
|
|
69
|
+
}
|
|
70
|
+
else if (current.pdfStatus === 'missing' && exists) {
|
|
71
|
+
task.states[paperId] = { ...current, pdfStatus: 'downloaded' };
|
|
72
|
+
changed = true;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return changed;
|
|
76
|
+
}
|
|
34
77
|
export class TaskService {
|
|
35
78
|
ready;
|
|
36
79
|
store = new TaskStore();
|
|
@@ -63,6 +106,8 @@ export class TaskService {
|
|
|
63
106
|
catch (error) {
|
|
64
107
|
console.warn('[dsh-plugin-litsearch] 任务目录加载失败:', error);
|
|
65
108
|
}
|
|
109
|
+
// 与任务自动清理同批执行:按总量上限修剪 PDF 去重缓存(0 表示关闭)。
|
|
110
|
+
await this.trimPdfCache();
|
|
66
111
|
let restored = false;
|
|
67
112
|
try {
|
|
68
113
|
const snapshot = await this.store.loadCurrent();
|
|
@@ -79,6 +124,21 @@ export class TaskService {
|
|
|
79
124
|
if (!restored)
|
|
80
125
|
this.bootstrapCurrentTask();
|
|
81
126
|
}
|
|
127
|
+
/** 按设置的总量上限修剪 PDF 去重缓存;失败只告警,不阻断启动。 */
|
|
128
|
+
async trimPdfCache() {
|
|
129
|
+
const maxMb = getSettings().pdfCacheMaxMb;
|
|
130
|
+
if (maxMb <= 0)
|
|
131
|
+
return;
|
|
132
|
+
try {
|
|
133
|
+
const outcome = await prunePdfCache(this.store.root, maxMb * 1024 * 1024);
|
|
134
|
+
if (outcome.removed > 0) {
|
|
135
|
+
console.info(`[dsh-plugin-litsearch] PDF 缓存已修剪:删除 ${outcome.removed} 个文件,释放 ${Math.round(outcome.freedBytes / 1024 / 1024)} MB(上限 ${maxMb} MB)`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
console.warn('[dsh-plugin-litsearch] PDF 缓存修剪失败:', error);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
82
142
|
/** 没有可用快照时,把最近更新的任务设为当前任务并落盘。 */
|
|
83
143
|
bootstrapCurrentTask() {
|
|
84
144
|
const latest = this.store.list()[0];
|
|
@@ -109,7 +169,11 @@ export class TaskService {
|
|
|
109
169
|
globalRev: this.events.currentRev,
|
|
110
170
|
sources: [...PROVIDER_INFOS],
|
|
111
171
|
sourceHealth: sourceHealthList(),
|
|
112
|
-
keys: {
|
|
172
|
+
keys: {
|
|
173
|
+
ncbi: settings.ncbiApiKey !== '',
|
|
174
|
+
s2: settings.s2ApiKey !== '',
|
|
175
|
+
serpapi: settings.serpApiKey !== '',
|
|
176
|
+
},
|
|
113
177
|
settings: {
|
|
114
178
|
defaultSources: [...settings.defaultSources],
|
|
115
179
|
defaultPageSize: settings.defaultPageSize,
|
|
@@ -119,7 +183,7 @@ export class TaskService {
|
|
|
119
183
|
maxTaskTabs: settings.maxTaskTabs,
|
|
120
184
|
},
|
|
121
185
|
fullText: {
|
|
122
|
-
pdfRoot: resolvePdfRoot(),
|
|
186
|
+
pdfRoot: settings.pdfRoot.trim() === '' ? '' : resolvePdfRoot(),
|
|
123
187
|
pdfMaxMb: settings.pdfMaxMb,
|
|
124
188
|
autoDownload: settings.autoDownloadPdf,
|
|
125
189
|
epmcFulltext: settings.enableEpmcFulltext,
|
|
@@ -161,8 +225,12 @@ export class TaskService {
|
|
|
161
225
|
task.rounds = [];
|
|
162
226
|
return task;
|
|
163
227
|
}
|
|
164
|
-
async getDetail(id) {
|
|
165
|
-
|
|
228
|
+
async getDetail(id, workspace) {
|
|
229
|
+
const task = await this.get(id);
|
|
230
|
+
if (await reconcilePdfFiles(task))
|
|
231
|
+
await this.store.save(task);
|
|
232
|
+
const target = normalizeWorkspace(workspace) ?? normalizeWorkspace(task.workspace);
|
|
233
|
+
return { ...toDetail(task), downloadDir: resolvePdfRoot(target) };
|
|
166
234
|
}
|
|
167
235
|
/** 任务内轮次历史(时间升序、深拷贝)。 */
|
|
168
236
|
async getRounds(id) {
|
|
@@ -206,11 +274,20 @@ export class TaskService {
|
|
|
206
274
|
});
|
|
207
275
|
}
|
|
208
276
|
/** 用户从任务列表点开某任务 = 进入该任务,之后的模型检索默认落在这里。 */
|
|
209
|
-
async activate(id) {
|
|
277
|
+
async activate(id, workspace) {
|
|
210
278
|
const task = await this.get(id);
|
|
279
|
+
await this.applyWorkspace(task, workspace);
|
|
211
280
|
this.setCurrentTask(task.id);
|
|
212
281
|
return task;
|
|
213
282
|
}
|
|
283
|
+
/** 记下任务最近一次所在的工作目录;只改变下载落点,不改变 rev。 */
|
|
284
|
+
async applyWorkspace(task, workspace) {
|
|
285
|
+
const next = normalizeWorkspace(workspace);
|
|
286
|
+
if (next === undefined || next === task.workspace)
|
|
287
|
+
return;
|
|
288
|
+
task.workspace = next;
|
|
289
|
+
await this.store.save(task);
|
|
290
|
+
}
|
|
214
291
|
/** 等待该任务当前这一轮结束;返回是否确实已空闲。 */
|
|
215
292
|
async waitUntilIdle(id, timeoutMs, signal) {
|
|
216
293
|
if (!this.runner.isRunning(id))
|
|
@@ -250,6 +327,7 @@ export class TaskService {
|
|
|
250
327
|
const conditions = this.readConditions(input, settings);
|
|
251
328
|
const title = readTitle(input.title, conditions);
|
|
252
329
|
const inherited = input.fromTaskId === undefined ? undefined : await this.get(input.fromTaskId);
|
|
330
|
+
const workspace = normalizeWorkspace(input.workspace);
|
|
253
331
|
const now = new Date().toISOString();
|
|
254
332
|
const task = {
|
|
255
333
|
id: `t${Date.now().toString(36)}${randomUUID().slice(0, 6)}`,
|
|
@@ -272,6 +350,7 @@ export class TaskService {
|
|
|
272
350
|
result: { papers: [], sources: [] },
|
|
273
351
|
rounds: [],
|
|
274
352
|
states: inherited === undefined ? {} : inheritStates(inherited),
|
|
353
|
+
...(workspace === undefined ? {} : { workspace }),
|
|
275
354
|
...(input.fromTaskId === undefined ? {} : { fromTaskId: input.fromTaskId }),
|
|
276
355
|
};
|
|
277
356
|
this.store.put(task);
|
|
@@ -342,17 +421,17 @@ export class TaskService {
|
|
|
342
421
|
*
|
|
343
422
|
* 任务已缓存时同步登记运行态,调用方紧接着调 `waitUntilIdle` 也不会误判为空闲。
|
|
344
423
|
*/
|
|
345
|
-
run(id, mode = 'new', origin = 'panel') {
|
|
424
|
+
run(id, mode = 'new', origin = 'panel', workspace) {
|
|
346
425
|
const cached = this.store.cached(id);
|
|
347
426
|
if (cached !== undefined) {
|
|
348
427
|
try {
|
|
349
|
-
return this.begin(cached, mode, origin);
|
|
428
|
+
return this.begin(cached, mode, origin, workspace);
|
|
350
429
|
}
|
|
351
430
|
catch (error) {
|
|
352
431
|
return Promise.reject(error);
|
|
353
432
|
}
|
|
354
433
|
}
|
|
355
|
-
return this.get(id).then((task) => this.begin(task, mode, origin));
|
|
434
|
+
return this.get(id).then((task) => this.begin(task, mode, origin, workspace));
|
|
356
435
|
}
|
|
357
436
|
/**
|
|
358
437
|
* 在同一任务内开新一轮:全量替换检索条件后执行,**不新建任务、不新开 tab**。
|
|
@@ -369,6 +448,9 @@ export class TaskService {
|
|
|
369
448
|
if (nextTitle !== '')
|
|
370
449
|
task.title = nextTitle.slice(0, MAX_TITLE);
|
|
371
450
|
task.conditions = this.readConditions(input, getSettings());
|
|
451
|
+
const workspace = normalizeWorkspace(input.workspace);
|
|
452
|
+
if (workspace !== undefined)
|
|
453
|
+
task.workspace = workspace;
|
|
372
454
|
task.rev += 1;
|
|
373
455
|
task.updatedAt = new Date().toISOString();
|
|
374
456
|
await this.store.save(task);
|
|
@@ -409,7 +491,7 @@ export class TaskService {
|
|
|
409
491
|
const resolved = base === undefined ? filter : await this.withBaseFilter(task, filter);
|
|
410
492
|
return computeStats(task, resolved, scope, base);
|
|
411
493
|
}
|
|
412
|
-
/** 写入筛读 /
|
|
494
|
+
/** 写入筛读 / 分组;返回新的 rev 与命中条数。 */
|
|
413
495
|
async annotate(id, inputs, expectedRev) {
|
|
414
496
|
const task = await this.get(id);
|
|
415
497
|
if (expectedRev !== undefined && expectedRev !== task.rev) {
|
|
@@ -483,10 +565,34 @@ export class TaskService {
|
|
|
483
565
|
remaining: this.store.list().length,
|
|
484
566
|
};
|
|
485
567
|
}
|
|
568
|
+
/**
|
|
569
|
+
* 「查看已下载」前的存在性检查:页面加载后文件可能被删除或移走。
|
|
570
|
+
* 缺失时把状态改为 `missing` 并广播,所有面板随即显示「下载文件已删除」。
|
|
571
|
+
*/
|
|
572
|
+
async recheckPdf(id, paperId) {
|
|
573
|
+
const task = await this.get(id);
|
|
574
|
+
findPaper(task, paperId);
|
|
575
|
+
const state = task.states[paperId];
|
|
576
|
+
const filePath = state?.pdfFile;
|
|
577
|
+
if (state === undefined || filePath === undefined)
|
|
578
|
+
return { exists: false };
|
|
579
|
+
pdfExistsCache.delete(filePath);
|
|
580
|
+
const exists = await pdfFileExists(filePath);
|
|
581
|
+
if (exists && state.pdfStatus !== 'downloaded') {
|
|
582
|
+
await this.setPdfState(task, paperId, { pdfStatus: 'downloaded', pdfFile: filePath });
|
|
583
|
+
}
|
|
584
|
+
else if (!exists && state.pdfStatus !== 'missing') {
|
|
585
|
+
await this.setPdfState(task, paperId, { pdfStatus: 'missing', pdfFile: filePath });
|
|
586
|
+
}
|
|
587
|
+
return { exists };
|
|
588
|
+
}
|
|
486
589
|
/** 下载单篇 PDF;失败时写入失败状态,若有 EPMC 全文则降级为 html-only。 */
|
|
487
|
-
async downloadPdf(id, paperId) {
|
|
590
|
+
async downloadPdf(id, paperId, workspace) {
|
|
488
591
|
const task = await this.get(id);
|
|
489
592
|
const paper = findPaper(task, paperId);
|
|
593
|
+
const target = normalizeWorkspace(workspace) ?? normalizeWorkspace(task.workspace);
|
|
594
|
+
if (target !== undefined)
|
|
595
|
+
task.workspace = target;
|
|
490
596
|
const settings = getSettings();
|
|
491
597
|
const resolution = await resolvePdf(paper);
|
|
492
598
|
if (!resolution.ok || resolution.buffer === undefined) {
|
|
@@ -500,10 +606,11 @@ export class TaskService {
|
|
|
500
606
|
status: fallback,
|
|
501
607
|
bytes: 0,
|
|
502
608
|
error: resolution.error ?? '下载失败',
|
|
609
|
+
...(resolution.reason === undefined ? {} : { reason: resolution.reason }),
|
|
503
610
|
};
|
|
504
611
|
return result;
|
|
505
612
|
}
|
|
506
|
-
const saved = await persistPdf(resolution.buffer, this.store.root, paper);
|
|
613
|
+
const saved = await persistPdf(resolution.buffer, this.store.root, paper, target);
|
|
507
614
|
await this.setPdfState(task, paperId, {
|
|
508
615
|
pdfStatus: 'downloaded',
|
|
509
616
|
pdfFile: saved.filePath,
|
|
@@ -519,20 +626,6 @@ export class TaskService {
|
|
|
519
626
|
result.url = resolution.url;
|
|
520
627
|
return result;
|
|
521
628
|
}
|
|
522
|
-
/** PDF 预览:优先用已落地文件;没有则临时抓取一次但不落盘。 */
|
|
523
|
-
async previewPdf(id, paperId) {
|
|
524
|
-
const task = await this.get(id);
|
|
525
|
-
const paper = findPaper(task, paperId);
|
|
526
|
-
const filename = readableFilename(paper);
|
|
527
|
-
const local = await readLocalPdf(paper);
|
|
528
|
-
if (local !== null)
|
|
529
|
-
return { buffer: local, filename };
|
|
530
|
-
const resolution = await resolvePdf(paper);
|
|
531
|
-
if (!resolution.ok || resolution.buffer === undefined) {
|
|
532
|
-
throw new SearchError('not_found', resolution.error ?? '没有可预览的 PDF');
|
|
533
|
-
}
|
|
534
|
-
return { buffer: resolution.buffer, filename };
|
|
535
|
-
}
|
|
536
629
|
/** Europe PMC 全文(JATS → 只含安全标签的 HTML)。 */
|
|
537
630
|
async fullText(id, paperId) {
|
|
538
631
|
const task = await this.get(id);
|
|
@@ -554,15 +647,19 @@ export class TaskService {
|
|
|
554
647
|
return html;
|
|
555
648
|
}
|
|
556
649
|
/** 单篇论文详情:独立论文页使用,含任务标题与分组。 */
|
|
557
|
-
async paperDetail(id, paperId) {
|
|
650
|
+
async paperDetail(id, paperId, workspace) {
|
|
558
651
|
const task = await this.get(id);
|
|
652
|
+
if (await reconcilePdfFiles(task))
|
|
653
|
+
await this.store.save(task);
|
|
559
654
|
const paper = findPaper(task, paperId);
|
|
655
|
+
const target = normalizeWorkspace(workspace) ?? normalizeWorkspace(task.workspace);
|
|
560
656
|
const detail = {
|
|
561
657
|
taskId: task.id,
|
|
562
658
|
taskTitle: task.title,
|
|
563
659
|
rev: task.rev,
|
|
564
660
|
paper,
|
|
565
661
|
groups: task.groups,
|
|
662
|
+
downloadDir: resolvePdfRoot(target),
|
|
566
663
|
};
|
|
567
664
|
const state = task.states[paperId];
|
|
568
665
|
if (state !== undefined)
|
|
@@ -593,11 +690,12 @@ export class TaskService {
|
|
|
593
690
|
}
|
|
594
691
|
}
|
|
595
692
|
/** 把选中论文同步到 Zotero(含本地 PDF 附件)。 */
|
|
596
|
-
async zoteroSync(id, paperIds) {
|
|
693
|
+
async zoteroSync(id, paperIds, workspace) {
|
|
597
694
|
const settings = getSettings();
|
|
598
695
|
if (!settings.zoteroEnabled)
|
|
599
696
|
throw new SearchError('bad_request', 'Zotero 同步已在设置中关闭');
|
|
600
697
|
const task = await this.get(id);
|
|
698
|
+
const target = normalizeWorkspace(workspace) ?? normalizeWorkspace(task.workspace);
|
|
601
699
|
const wanted = paperIds === undefined || paperIds.length === 0
|
|
602
700
|
? task.result.papers.slice(0, 50)
|
|
603
701
|
: paperIds.map((paperId) => findPaper(task, paperId));
|
|
@@ -612,8 +710,13 @@ export class TaskService {
|
|
|
612
710
|
const attachPdf = settings.zoteroAttachPdf && !readOnlyAttachments;
|
|
613
711
|
const inputs = [];
|
|
614
712
|
for (const paper of wanted) {
|
|
615
|
-
const
|
|
616
|
-
|
|
713
|
+
const state = task.states[paper.paperId];
|
|
714
|
+
// 标签已移除,用任务内的分组名承载 Zotero 侧的分类信息。
|
|
715
|
+
const group = task.groups.find((entry) => entry.id === state?.groupId);
|
|
716
|
+
const tags = group === undefined ? [] : [group.name];
|
|
717
|
+
const pdfPath = attachPdf
|
|
718
|
+
? await this.ensureLocalPdf(paper, target, state?.pdfFile)
|
|
719
|
+
: undefined;
|
|
617
720
|
inputs.push({ paper, tags, ...(pdfPath === undefined ? {} : { pdfPath }) });
|
|
618
721
|
}
|
|
619
722
|
try {
|
|
@@ -635,8 +738,8 @@ export class TaskService {
|
|
|
635
738
|
}
|
|
636
739
|
}
|
|
637
740
|
/** 下载目录可写性测试(设置页)。 */
|
|
638
|
-
async storageProbe() {
|
|
639
|
-
const outcome = await probePdfRoot();
|
|
741
|
+
async storageProbe(workspace) {
|
|
742
|
+
const outcome = await probePdfRoot(workspace);
|
|
640
743
|
if (outcome.writable)
|
|
641
744
|
return { ok: true, root: outcome.root, writable: true };
|
|
642
745
|
return {
|
|
@@ -646,14 +749,19 @@ export class TaskService {
|
|
|
646
749
|
...(outcome.error === undefined ? {} : { error: outcome.error }),
|
|
647
750
|
};
|
|
648
751
|
}
|
|
649
|
-
async ensureLocalPdf(paper) {
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
752
|
+
async ensureLocalPdf(paper, workspace, recordedPath) {
|
|
753
|
+
if (recordedPath !== undefined) {
|
|
754
|
+
const recorded = await readSavedPdf(recordedPath);
|
|
755
|
+
if (recorded !== null)
|
|
756
|
+
return recordedPath;
|
|
757
|
+
}
|
|
758
|
+
const localPath = join(resolvePdfRoot(workspace), readableFilename(paper));
|
|
759
|
+
if ((await readSavedPdf(localPath)) !== null)
|
|
760
|
+
return localPath;
|
|
653
761
|
const resolution = await resolvePdf(paper).catch(() => null);
|
|
654
762
|
if (resolution === null || !resolution.ok || resolution.buffer === undefined)
|
|
655
763
|
return undefined;
|
|
656
|
-
const saved = await persistPdf(resolution.buffer, this.store.root, paper);
|
|
764
|
+
const saved = await persistPdf(resolution.buffer, this.store.root, paper, workspace);
|
|
657
765
|
return saved.filePath;
|
|
658
766
|
}
|
|
659
767
|
async setPdfState(task, paperId, patch) {
|
|
@@ -670,6 +778,10 @@ export class TaskService {
|
|
|
670
778
|
delete next.pdfFile;
|
|
671
779
|
else
|
|
672
780
|
next.pdfFile = patch.pdfFile.slice(0, 1024);
|
|
781
|
+
// 刚写入的文件立即视为存在,避免存在性缓存把状态又翻回 `missing`。
|
|
782
|
+
if (patch.pdfStatus === 'downloaded' && next.pdfFile !== undefined) {
|
|
783
|
+
pdfExistsCache.set(next.pdfFile, { exists: true, at: Date.now() });
|
|
784
|
+
}
|
|
673
785
|
task.states[paperId] = next;
|
|
674
786
|
task.rev += 1;
|
|
675
787
|
task.updatedAt = new Date().toISOString();
|
|
@@ -744,10 +856,15 @@ export class TaskService {
|
|
|
744
856
|
this.runner.cancelAll();
|
|
745
857
|
void this.store.flush();
|
|
746
858
|
}
|
|
747
|
-
begin(task, mode, origin = task.origin) {
|
|
859
|
+
begin(task, mode, origin = task.origin, workspace) {
|
|
748
860
|
if (this.runner.isRunning(task.id)) {
|
|
749
861
|
throw new SearchError('conflict', '任务正在运行');
|
|
750
862
|
}
|
|
863
|
+
const nextWorkspace = normalizeWorkspace(workspace);
|
|
864
|
+
if (nextWorkspace !== undefined && nextWorkspace !== task.workspace) {
|
|
865
|
+
task.workspace = nextWorkspace;
|
|
866
|
+
void this.store.save(task);
|
|
867
|
+
}
|
|
751
868
|
if (mode === 'more') {
|
|
752
869
|
throw new SearchError('bad_request', '续页加载将在后续版本提供,请先重跑任务');
|
|
753
870
|
}
|
|
@@ -796,7 +913,7 @@ export class TaskService {
|
|
|
796
913
|
}
|
|
797
914
|
continue;
|
|
798
915
|
}
|
|
799
|
-
const saved = await persistPdf(resolution.buffer, this.store.root, paper);
|
|
916
|
+
const saved = await persistPdf(resolution.buffer, this.store.root, paper, task.workspace);
|
|
800
917
|
await this.setPdfState(task, paper.paperId, {
|
|
801
918
|
pdfStatus: 'downloaded',
|
|
802
919
|
pdfFile: saved.filePath,
|
|
@@ -914,14 +1031,8 @@ function readTitle(title, conditions) {
|
|
|
914
1031
|
function inheritStates(base) {
|
|
915
1032
|
const states = {};
|
|
916
1033
|
for (const [paperId, state] of Object.entries(base.states)) {
|
|
917
|
-
//
|
|
1034
|
+
// 分组是任务内概念,继承时只带筛读状态。
|
|
918
1035
|
const next = { screening: state.screening };
|
|
919
|
-
if (state.shortlistOrder !== undefined)
|
|
920
|
-
next.shortlistOrder = state.shortlistOrder;
|
|
921
|
-
if (state.note !== undefined)
|
|
922
|
-
next.note = state.note;
|
|
923
|
-
if (state.tags !== undefined)
|
|
924
|
-
next.tags = [...state.tags];
|
|
925
1036
|
states[paperId] = next;
|
|
926
1037
|
}
|
|
927
1038
|
return states;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* 筛读台账:论文级状态(四态筛读 /
|
|
2
|
+
* 筛读台账:论文级状态(四态筛读 / 分组)与任务分组。
|
|
3
3
|
*
|
|
4
4
|
* 所有写入口(面板注解、模型 `literature_task annotate`、分组编辑)都先经过
|
|
5
5
|
* 这里的校验,确保状态结构稳定;`states` 以稳定的 `paperId` 为键,因此跨任务
|
|
@@ -24,5 +24,3 @@ export declare function applyAnnotations(task: SearchTask, inputs: readonly Anno
|
|
|
24
24
|
export declare function normalizeGroups(values: readonly TaskGroup[] | undefined): TaskGroup[] | undefined;
|
|
25
25
|
/** 分组若仍有论文引用,删除前提醒调用方(调用方决定是否允许)。 */
|
|
26
26
|
export declare function groupUsage(task: SearchTask, groupId: string): number;
|
|
27
|
-
/** 按必读顺序、年份、标题排序某组的论文。 */
|
|
28
|
-
export declare function sortGroupPapers(task: SearchTask, papers: readonly Paper[]): Paper[];
|
package/lib/host/tasks/states.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* 筛读台账:论文级状态(四态筛读 /
|
|
2
|
+
* 筛读台账:论文级状态(四态筛读 / 分组)与任务分组。
|
|
3
3
|
*
|
|
4
4
|
* 所有写入口(面板注解、模型 `literature_task annotate`、分组编辑)都先经过
|
|
5
5
|
* 这里的校验,确保状态结构稳定;`states` 以稳定的 `paperId` 为键,因此跨任务
|
|
@@ -10,17 +10,14 @@ const MAX_ANNOTATIONS = 50;
|
|
|
10
10
|
const MAX_GROUPS = 50;
|
|
11
11
|
const MAX_GROUP_NAME = 60;
|
|
12
12
|
const MAX_GROUP_DESCRIPTION = 500;
|
|
13
|
-
const MAX_NOTE = 4000;
|
|
14
|
-
const MAX_TAGS = 20;
|
|
15
|
-
const MAX_TAG = 40;
|
|
16
13
|
const MAX_PDF_ERROR = 300;
|
|
17
14
|
const MAX_PDF_FILE = 1024;
|
|
18
|
-
const MAX_SHORTLIST_ORDER = 100_000;
|
|
19
15
|
export const SCREENINGS = ['unread', 'read', 'included', 'excluded'];
|
|
20
16
|
export const PDF_STATUSES = [
|
|
21
17
|
'unknown',
|
|
22
18
|
'available',
|
|
23
19
|
'downloaded',
|
|
20
|
+
'missing',
|
|
24
21
|
'html-only',
|
|
25
22
|
'paywalled',
|
|
26
23
|
'blocked',
|
|
@@ -86,40 +83,6 @@ export function applyAnnotations(task, inputs) {
|
|
|
86
83
|
}
|
|
87
84
|
touched = true;
|
|
88
85
|
}
|
|
89
|
-
if (input.shortlistOrder !== undefined) {
|
|
90
|
-
if (input.shortlistOrder === null) {
|
|
91
|
-
delete next.shortlistOrder;
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
if (!Number.isInteger(input.shortlistOrder) ||
|
|
95
|
-
input.shortlistOrder < 0 ||
|
|
96
|
-
input.shortlistOrder > MAX_SHORTLIST_ORDER) {
|
|
97
|
-
throw new SearchError('bad_request', `shortlistOrder 必须是 0..${MAX_SHORTLIST_ORDER} 的整数`);
|
|
98
|
-
}
|
|
99
|
-
next.shortlistOrder = input.shortlistOrder;
|
|
100
|
-
}
|
|
101
|
-
touched = true;
|
|
102
|
-
}
|
|
103
|
-
if (input.note !== undefined) {
|
|
104
|
-
if (typeof input.note !== 'string')
|
|
105
|
-
throw new SearchError('bad_request', 'note 必须是字符串');
|
|
106
|
-
const note = input.note.trim();
|
|
107
|
-
if (note === '')
|
|
108
|
-
delete next.note;
|
|
109
|
-
else
|
|
110
|
-
next.note = note.slice(0, MAX_NOTE);
|
|
111
|
-
touched = true;
|
|
112
|
-
}
|
|
113
|
-
if (input.tags !== undefined) {
|
|
114
|
-
if (!Array.isArray(input.tags))
|
|
115
|
-
throw new SearchError('bad_request', 'tags 必须是字符串数组');
|
|
116
|
-
const tags = normalizeTags(input.tags);
|
|
117
|
-
if (tags.length === 0)
|
|
118
|
-
delete next.tags;
|
|
119
|
-
else
|
|
120
|
-
next.tags = tags;
|
|
121
|
-
touched = true;
|
|
122
|
-
}
|
|
123
86
|
if (input.pdfStatus !== undefined) {
|
|
124
87
|
if (!isPdfStatus(input.pdfStatus)) {
|
|
125
88
|
throw new SearchError('bad_request', `pdfStatus 只能是 ${PDF_STATUSES.join(' / ')}`);
|
|
@@ -194,30 +157,3 @@ export function normalizeGroups(values) {
|
|
|
194
157
|
export function groupUsage(task, groupId) {
|
|
195
158
|
return Object.values(task.states).filter((state) => state.groupId === groupId).length;
|
|
196
159
|
}
|
|
197
|
-
/** 按必读顺序、年份、标题排序某组的论文。 */
|
|
198
|
-
export function sortGroupPapers(task, papers) {
|
|
199
|
-
return [...papers].sort((a, b) => {
|
|
200
|
-
const left = stateOf(task, a.paperId).shortlistOrder;
|
|
201
|
-
const right = stateOf(task, b.paperId).shortlistOrder;
|
|
202
|
-
if (left !== undefined || right !== undefined) {
|
|
203
|
-
if (left === undefined)
|
|
204
|
-
return 1;
|
|
205
|
-
if (right === undefined)
|
|
206
|
-
return -1;
|
|
207
|
-
if (left !== right)
|
|
208
|
-
return left - right;
|
|
209
|
-
}
|
|
210
|
-
return (b.year ?? 0) - (a.year ?? 0) || a.title.localeCompare(b.title);
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
function normalizeTags(values) {
|
|
214
|
-
const seen = new Set();
|
|
215
|
-
for (const raw of values) {
|
|
216
|
-
const tag = raw.trim().slice(0, MAX_TAG);
|
|
217
|
-
if (tag !== '')
|
|
218
|
-
seen.add(tag);
|
|
219
|
-
if (seen.size >= MAX_TAGS)
|
|
220
|
-
break;
|
|
221
|
-
}
|
|
222
|
-
return [...seen];
|
|
223
|
-
}
|
package/lib/host/tasks/stats.js
CHANGED
|
@@ -10,6 +10,8 @@ const CACHE_TTL_MS = 30_000;
|
|
|
10
10
|
const CACHE_MAX = 16;
|
|
11
11
|
const TOP_LIST = 15;
|
|
12
12
|
const TOP_PAPERS = 10;
|
|
13
|
+
/** 出处 / 作者候选值的上限:筛选下拉直接用这份清单,按命中数取前 500。 */
|
|
14
|
+
const FACET_LIST = 500;
|
|
13
15
|
const cache = new Map();
|
|
14
16
|
export function computeStats(task, filter, scope, base) {
|
|
15
17
|
const key = `${task.id}:${task.rev}:${scope}:${filterKey(filter)}:${base?.rev ?? ''}`;
|
|
@@ -53,8 +55,8 @@ function buildReport(task, filter, scope, base) {
|
|
|
53
55
|
byYear: buildYearStats(task, papers),
|
|
54
56
|
byType: countBy(task, papers, (paper) => paper.type),
|
|
55
57
|
byLanguage: countBy(task, papers, (paper) => paper.language ?? ''),
|
|
56
|
-
topVenues: topBy(task, papers, (paper) => (paper.venue === '' ? [] : [paper.venue])),
|
|
57
|
-
topAuthors: topBy(task, papers, (paper) => paper.authors),
|
|
58
|
+
topVenues: topBy(task, papers, (paper) => (paper.venue === '' ? [] : [paper.venue]), FACET_LIST),
|
|
59
|
+
topAuthors: topBy(task, papers, (paper) => paper.authors, FACET_LIST),
|
|
58
60
|
metrics: buildMetrics(papers, base),
|
|
59
61
|
screeningProgress: { included, excluded, pending: unread },
|
|
60
62
|
unavailableFullText: {
|
|
@@ -70,6 +72,7 @@ function countFullText(task, papers) {
|
|
|
70
72
|
unknown: 0,
|
|
71
73
|
available: 0,
|
|
72
74
|
downloaded: 0,
|
|
75
|
+
missing: 0,
|
|
73
76
|
htmlOnly: 0,
|
|
74
77
|
paywalled: 0,
|
|
75
78
|
blocked: 0,
|
|
@@ -83,6 +86,8 @@ function countFullText(task, papers) {
|
|
|
83
86
|
coverage.available += 1;
|
|
84
87
|
else if (status === 'downloaded')
|
|
85
88
|
coverage.downloaded += 1;
|
|
89
|
+
else if (status === 'missing')
|
|
90
|
+
coverage.missing += 1;
|
|
86
91
|
else if (status === 'paywalled')
|
|
87
92
|
coverage.paywalled += 1;
|
|
88
93
|
else if (status === 'blocked')
|
|
@@ -175,7 +180,7 @@ function countBy(task, papers, keyOf) {
|
|
|
175
180
|
.map(([key, entry]) => ({ key, ...entry }))
|
|
176
181
|
.sort((a, b) => b.count - a.count);
|
|
177
182
|
}
|
|
178
|
-
function topBy(task, papers, keysOf) {
|
|
183
|
+
function topBy(task, papers, keysOf, limit) {
|
|
179
184
|
const counts = new Map();
|
|
180
185
|
for (const paper of papers) {
|
|
181
186
|
const included = screeningOf(task, paper) === 'included';
|
|
@@ -193,7 +198,7 @@ function topBy(task, papers, keysOf) {
|
|
|
193
198
|
return [...counts.entries()]
|
|
194
199
|
.map(([key, entry]) => ({ key, ...entry }))
|
|
195
200
|
.sort((a, b) => b.count - a.count)
|
|
196
|
-
.slice(0,
|
|
201
|
+
.slice(0, limit);
|
|
197
202
|
}
|
|
198
203
|
function buildMetrics(papers, base) {
|
|
199
204
|
const total = papers.length;
|
|
@@ -47,7 +47,7 @@ export declare class TaskStore {
|
|
|
47
47
|
/** 读取「当前任务」快照;文件缺失或损坏时返回 null(由调用方自举)。 */
|
|
48
48
|
loadCurrent(): Promise<CurrentTaskSnapshot | null>;
|
|
49
49
|
saveCurrent(snapshot: CurrentTaskSnapshot): Promise<void>;
|
|
50
|
-
/**
|
|
50
|
+
/** 手动清理:删除所有未保留任务,返回条数与释放的字节数。 */
|
|
51
51
|
cleanupUnpinned(): Promise<{
|
|
52
52
|
removed: number;
|
|
53
53
|
freedBytes: number;
|
package/lib/host/tasks/store.js
CHANGED
|
@@ -135,7 +135,7 @@ export class TaskStore {
|
|
|
135
135
|
await atomicWrite(this.currentFile, body);
|
|
136
136
|
});
|
|
137
137
|
}
|
|
138
|
-
/**
|
|
138
|
+
/** 手动清理:删除所有未保留任务,返回条数与释放的字节数。 */
|
|
139
139
|
async cleanupUnpinned() {
|
|
140
140
|
const targets = this.list().filter((summary) => !summary.pinned);
|
|
141
141
|
let freedBytes = 0;
|
|
@@ -27,7 +27,7 @@ export function toSummary(task) {
|
|
|
27
27
|
export function toDetail(task) {
|
|
28
28
|
const states = {};
|
|
29
29
|
for (const [paperId, state] of Object.entries(task.states)) {
|
|
30
|
-
states[paperId] = { ...state
|
|
30
|
+
states[paperId] = { ...state };
|
|
31
31
|
}
|
|
32
32
|
return {
|
|
33
33
|
...toSummary(task),
|
package/lib/host/tools.d.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* 都有效);只有用户明确要求新任务时才新建。等待最多 25 秒后把任务头部与前
|
|
7
7
|
* 若干条结果交给模型,并明确要求本轮到此为止、只给完善建议、不发起下一轮。
|
|
8
8
|
* - `literature_sources`:列出已接入的数据源与能力,便于模型决定后续策略。
|
|
9
|
-
* - `literature_task
|
|
9
|
+
* - `literature_task`:读取/修改已保存任务(含轮次历史),写入筛读与分组,
|
|
10
10
|
* 做新老对比与统计。
|
|
11
11
|
* - `literature_export`:把任务(按筛选)导出为 CSV / RIS / BibTeX / Markdown 并落盘。
|
|
12
12
|
*
|