dsh-plugin-litsearch 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -13
- package/client/client.js +4397 -1953
- package/lib/host/events.d.ts +3 -0
- package/lib/host/events.js +25 -1
- package/lib/host/paths.d.ts +44 -0
- package/lib/host/paths.js +129 -0
- package/lib/host/pdf/root.d.ts +11 -0
- package/lib/host/pdf/root.js +27 -0
- package/lib/host/pdf/store.d.ts +13 -5
- package/lib/host/pdf/store.js +62 -17
- package/lib/host/router.js +35 -11
- package/lib/host/tasks/filter.d.ts +5 -1
- package/lib/host/tasks/filter.js +49 -4
- package/lib/host/tasks/runner.js +1 -8
- package/lib/host/tasks/service.d.ts +34 -15
- package/lib/host/tasks/service.js +180 -53
- package/lib/host/tasks/stats.js +6 -4
- package/lib/host/tasks/store.d.ts +24 -4
- package/lib/host/tasks/store.js +64 -6
- package/lib/host/tools.d.ts +13 -6
- package/lib/host/tools.js +73 -29
- package/lib/host/workspace.d.ts +20 -0
- package/lib/host/workspace.js +32 -0
- package/lib/index.d.ts +17 -8
- package/lib/index.js +33 -11
- package/lib/search/aggregate.js +1 -0
- package/lib/search/config.d.ts +17 -8
- package/lib/search/config.js +16 -8
- package/lib/search/identity.js +1 -0
- package/lib/search/providers/googlescholar.d.ts +12 -0
- package/lib/search/providers/googlescholar.js +154 -0
- package/lib/search/providers/index.js +2 -0
- package/lib/search/query-adapter.js +3 -0
- package/lib/search/rate-limit.js +2 -1
- package/lib/search/types.d.ts +27 -5
- package/lib/search/types.js +21 -1
- package/package.json +1 -1
package/lib/host/events.d.ts
CHANGED
|
@@ -9,8 +9,11 @@ import type { EventsPage, TaskSummary } from '../search/types.js';
|
|
|
9
9
|
export declare class TaskEvents {
|
|
10
10
|
private rev;
|
|
11
11
|
private readonly entries;
|
|
12
|
+
private focus;
|
|
12
13
|
get currentRev(): number;
|
|
13
14
|
publish(summary: TaskSummary): void;
|
|
15
|
+
/** 请求客户端把某个任务 tab 打开/切到前台(面板关着时重新打开)。 */
|
|
16
|
+
requestFocus(taskId: string, title: string, sessionId?: string): void;
|
|
14
17
|
remove(id: string): void;
|
|
15
18
|
since(sinceRev: number): EventsPage;
|
|
16
19
|
}
|
package/lib/host/events.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
export class TaskEvents {
|
|
9
9
|
rev = 0;
|
|
10
10
|
entries = new Map();
|
|
11
|
+
focus;
|
|
11
12
|
get currentRev() {
|
|
12
13
|
return this.rev;
|
|
13
14
|
}
|
|
@@ -15,8 +16,21 @@ export class TaskEvents {
|
|
|
15
16
|
this.rev += 1;
|
|
16
17
|
this.entries.set(summary.id, { globalRev: this.rev, summary });
|
|
17
18
|
}
|
|
19
|
+
/** 请求客户端把某个任务 tab 打开/切到前台(面板关着时重新打开)。 */
|
|
20
|
+
requestFocus(taskId, title, sessionId) {
|
|
21
|
+
this.rev += 1;
|
|
22
|
+
this.focus = {
|
|
23
|
+
globalRev: this.rev,
|
|
24
|
+
taskId,
|
|
25
|
+
title,
|
|
26
|
+
at: Date.now(),
|
|
27
|
+
...(sessionId === undefined ? {} : { sessionId }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
18
30
|
remove(id) {
|
|
19
31
|
this.entries.delete(id);
|
|
32
|
+
if (this.focus?.taskId === id)
|
|
33
|
+
this.focus = undefined;
|
|
20
34
|
}
|
|
21
35
|
since(sinceRev) {
|
|
22
36
|
if (!Number.isFinite(sinceRev) || sinceRev < 0) {
|
|
@@ -29,6 +43,16 @@ export class TaskEvents {
|
|
|
29
43
|
.filter((entry) => entry.globalRev > sinceRev)
|
|
30
44
|
.sort((a, b) => a.globalRev - b.globalRev)
|
|
31
45
|
.map((entry) => entry.summary);
|
|
32
|
-
|
|
46
|
+
const focus = this.focus !== undefined && this.focus.globalRev > sinceRev
|
|
47
|
+
? {
|
|
48
|
+
taskId: this.focus.taskId,
|
|
49
|
+
title: this.focus.title,
|
|
50
|
+
at: this.focus.at,
|
|
51
|
+
...(this.focus.sessionId === undefined
|
|
52
|
+
? {}
|
|
53
|
+
: { sessionId: this.focus.sessionId }),
|
|
54
|
+
}
|
|
55
|
+
: undefined;
|
|
56
|
+
return { rev: this.rev, reset: false, tasks, ...(focus === undefined ? {} : { focus }) };
|
|
33
57
|
}
|
|
34
58
|
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 插件数据目录解析与旧版数据迁移。
|
|
3
|
+
*
|
|
4
|
+
* 平台约定:插件私有数据统一放在 `$DSH_HOME/storages/<plugin-id>/`,
|
|
5
|
+
* `DSH_HOME` 由宿主注入,未设置时回落 `~/.dsh`。本插件历史版本把数据直接
|
|
6
|
+
* 写在 `~/.dsh/dsh-litsearch/`,这里负责一次性搬迁到统一存储区:
|
|
7
|
+
*
|
|
8
|
+
* - 只复制、不删除,旧目录原样保留作备份;
|
|
9
|
+
* - 不覆盖新目录里已有的文件,重复启动不会反复迁移;
|
|
10
|
+
* - 复制失败或校验不通过时回退到旧目录读取,保证任务标注不丢。
|
|
11
|
+
*
|
|
12
|
+
* PDF 的可读文件由设置项 `pdfRoot` 决定(默认落在当前会话工作目录下的
|
|
13
|
+
* `dsh-litsearch/`),不受本模块影响;只有插件内部的去重缓存 `<root>/pdfs`
|
|
14
|
+
* 随数据目录一起搬迁。
|
|
15
|
+
*/
|
|
16
|
+
export declare const PLUGIN_ID = "dsh-plugin-litsearch";
|
|
17
|
+
/** 历史版本使用的目录名(`$DSH_HOME` 下的一级目录)。 */
|
|
18
|
+
export declare const LEGACY_DIR = "dsh-litsearch";
|
|
19
|
+
/** 解析宿主数据根目录;支持绝对路径、`~/...` 与空值回落。 */
|
|
20
|
+
export declare function dshHome(env?: NodeJS.ProcessEnv, home?: string): string;
|
|
21
|
+
/** 当前数据根目录:平台统一存储区下的插件域。 */
|
|
22
|
+
export declare function storageRoot(env?: NodeJS.ProcessEnv, home?: string): string;
|
|
23
|
+
/** 旧数据根目录候选,按优先级排列,已排除新根目录。 */
|
|
24
|
+
export declare function legacyRoots(env?: NodeJS.ProcessEnv, home?: string): string[];
|
|
25
|
+
/** 目录里是否存在至少一个任务正文文件(索引与快照不算)。 */
|
|
26
|
+
export declare function hasTaskFiles(root: string): Promise<boolean>;
|
|
27
|
+
export interface StorageResolution {
|
|
28
|
+
/** 最终使用的数据根目录。 */
|
|
29
|
+
root: string;
|
|
30
|
+
/** 本次复制的文件数。 */
|
|
31
|
+
copied: number;
|
|
32
|
+
/** 从该旧目录迁移成功。 */
|
|
33
|
+
migratedFrom?: string;
|
|
34
|
+
/** 迁移失败,暂时回退到该旧目录读取。 */
|
|
35
|
+
fallbackFrom?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* 决定最终数据根目录,并在必要时做一次性迁移:
|
|
39
|
+
*
|
|
40
|
+
* 1. 新目录已有任务 → 直接使用,绝不碰旧数据;
|
|
41
|
+
* 2. 新目录为空且旧目录有任务 → 复制缺失文件到新目录,旧目录保留;
|
|
42
|
+
* 3. 复制失败或复制后仍无任务 → 回滚本次副本并回退到旧目录读取。
|
|
43
|
+
*/
|
|
44
|
+
export declare function resolveStorageRoot(env?: NodeJS.ProcessEnv, home?: string): Promise<StorageResolution>;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 插件数据目录解析与旧版数据迁移。
|
|
3
|
+
*
|
|
4
|
+
* 平台约定:插件私有数据统一放在 `$DSH_HOME/storages/<plugin-id>/`,
|
|
5
|
+
* `DSH_HOME` 由宿主注入,未设置时回落 `~/.dsh`。本插件历史版本把数据直接
|
|
6
|
+
* 写在 `~/.dsh/dsh-litsearch/`,这里负责一次性搬迁到统一存储区:
|
|
7
|
+
*
|
|
8
|
+
* - 只复制、不删除,旧目录原样保留作备份;
|
|
9
|
+
* - 不覆盖新目录里已有的文件,重复启动不会反复迁移;
|
|
10
|
+
* - 复制失败或校验不通过时回退到旧目录读取,保证任务标注不丢。
|
|
11
|
+
*
|
|
12
|
+
* PDF 的可读文件由设置项 `pdfRoot` 决定(默认落在当前会话工作目录下的
|
|
13
|
+
* `dsh-litsearch/`),不受本模块影响;只有插件内部的去重缓存 `<root>/pdfs`
|
|
14
|
+
* 随数据目录一起搬迁。
|
|
15
|
+
*/
|
|
16
|
+
import { copyFile, mkdir, readdir, rm, stat } from 'node:fs/promises';
|
|
17
|
+
import { homedir } from 'node:os';
|
|
18
|
+
import { dirname, join, resolve } from 'node:path';
|
|
19
|
+
export const PLUGIN_ID = 'dsh-plugin-litsearch';
|
|
20
|
+
/** 历史版本使用的目录名(`$DSH_HOME` 下的一级目录)。 */
|
|
21
|
+
export const LEGACY_DIR = 'dsh-litsearch';
|
|
22
|
+
/** 允许搬迁的条目:任务正文与索引、导出文件、PDF 去重缓存、当前任务快照。 */
|
|
23
|
+
const MIGRATABLE_ENTRIES = ['tasks', 'out', 'pdfs', 'current.json'];
|
|
24
|
+
/** 解析宿主数据根目录;支持绝对路径、`~/...` 与空值回落。 */
|
|
25
|
+
export function dshHome(env = process.env, home = homedir()) {
|
|
26
|
+
const raw = env.DSH_HOME?.trim() ?? '';
|
|
27
|
+
if (raw === '')
|
|
28
|
+
return join(home, '.dsh');
|
|
29
|
+
if (raw === '~')
|
|
30
|
+
return home;
|
|
31
|
+
if (raw.startsWith('~/'))
|
|
32
|
+
return join(home, raw.slice(2));
|
|
33
|
+
// 绝对路径原样规范化,相对路径按进程工作目录解析(与宿主 path.resolve 一致)。
|
|
34
|
+
return resolve(raw);
|
|
35
|
+
}
|
|
36
|
+
/** 当前数据根目录:平台统一存储区下的插件域。 */
|
|
37
|
+
export function storageRoot(env = process.env, home = homedir()) {
|
|
38
|
+
return join(dshHome(env, home), 'storages', PLUGIN_ID);
|
|
39
|
+
}
|
|
40
|
+
/** 旧数据根目录候选,按优先级排列,已排除新根目录。 */
|
|
41
|
+
export function legacyRoots(env = process.env, home = homedir()) {
|
|
42
|
+
const current = storageRoot(env, home);
|
|
43
|
+
const candidates = [
|
|
44
|
+
join(dshHome(env, home), LEGACY_DIR), // 过渡形态:$DSH_HOME/dsh-litsearch
|
|
45
|
+
join(home, '.dsh', LEGACY_DIR), // 历史硬编码:~/.dsh/dsh-litsearch
|
|
46
|
+
];
|
|
47
|
+
return [...new Set(candidates)].filter((path) => path !== current);
|
|
48
|
+
}
|
|
49
|
+
/** 目录里是否存在至少一个任务正文文件(索引与快照不算)。 */
|
|
50
|
+
export async function hasTaskFiles(root) {
|
|
51
|
+
const names = await readdir(join(root, 'tasks')).catch(() => []);
|
|
52
|
+
return names.some((name) => name.endsWith('.json') && name !== 'index.json');
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* 决定最终数据根目录,并在必要时做一次性迁移:
|
|
56
|
+
*
|
|
57
|
+
* 1. 新目录已有任务 → 直接使用,绝不碰旧数据;
|
|
58
|
+
* 2. 新目录为空且旧目录有任务 → 复制缺失文件到新目录,旧目录保留;
|
|
59
|
+
* 3. 复制失败或复制后仍无任务 → 回滚本次副本并回退到旧目录读取。
|
|
60
|
+
*/
|
|
61
|
+
export async function resolveStorageRoot(env = process.env, home = homedir()) {
|
|
62
|
+
const root = storageRoot(env, home);
|
|
63
|
+
if (await hasTaskFiles(root))
|
|
64
|
+
return { root, copied: 0 };
|
|
65
|
+
for (const legacy of legacyRoots(env, home)) {
|
|
66
|
+
if (!(await hasTaskFiles(legacy)))
|
|
67
|
+
continue;
|
|
68
|
+
const copied = [];
|
|
69
|
+
try {
|
|
70
|
+
await copyLegacyStorage(legacy, root, copied);
|
|
71
|
+
if (await hasTaskFiles(root)) {
|
|
72
|
+
console.info(`[dsh-plugin-litsearch] 任务数据已迁移:${legacy} → ${root}(${copied.length} 个文件,原目录保留)`);
|
|
73
|
+
return { root, migratedFrom: legacy, copied: copied.length };
|
|
74
|
+
}
|
|
75
|
+
await rollback(copied);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
await rollback(copied);
|
|
79
|
+
console.warn('[dsh-plugin-litsearch] 任务数据迁移失败:', error);
|
|
80
|
+
}
|
|
81
|
+
console.warn(`[dsh-plugin-litsearch] 暂时沿用旧任务目录:${legacy}`);
|
|
82
|
+
return { root: legacy, fallbackFrom: legacy, copied: 0 };
|
|
83
|
+
}
|
|
84
|
+
return { root, copied: 0 };
|
|
85
|
+
}
|
|
86
|
+
/** 复制旧目录里的已知条目;目标已存在则跳过,复制路径写入 `copied` 供回滚。 */
|
|
87
|
+
async function copyLegacyStorage(from, to, copied) {
|
|
88
|
+
for (const entry of MIGRATABLE_ENTRIES) {
|
|
89
|
+
const source = join(from, entry);
|
|
90
|
+
const target = join(to, entry);
|
|
91
|
+
const info = await stat(source).catch(() => null);
|
|
92
|
+
if (info === null)
|
|
93
|
+
continue;
|
|
94
|
+
if (info.isFile()) {
|
|
95
|
+
await copyFileIfAbsent(source, target, copied);
|
|
96
|
+
}
|
|
97
|
+
else if (info.isDirectory()) {
|
|
98
|
+
await copyDirIfAbsent(source, target, copied);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async function copyDirIfAbsent(source, target, copied) {
|
|
103
|
+
const entries = await readdir(source, { withFileTypes: true });
|
|
104
|
+
for (const entry of entries) {
|
|
105
|
+
// 跳过原子写残留的临时文件与系统杂项。
|
|
106
|
+
if (entry.name.startsWith('.') || entry.name.endsWith('.tmp'))
|
|
107
|
+
continue;
|
|
108
|
+
const from = join(source, entry.name);
|
|
109
|
+
const to = join(target, entry.name);
|
|
110
|
+
if (entry.isDirectory()) {
|
|
111
|
+
await copyDirIfAbsent(from, to, copied);
|
|
112
|
+
}
|
|
113
|
+
else if (entry.isFile()) {
|
|
114
|
+
await copyFileIfAbsent(from, to, copied);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async function copyFileIfAbsent(source, target, copied) {
|
|
119
|
+
if ((await stat(target).catch(() => null)) !== null)
|
|
120
|
+
return;
|
|
121
|
+
await mkdir(dirname(target), { recursive: true });
|
|
122
|
+
await copyFile(source, target);
|
|
123
|
+
copied.push(target);
|
|
124
|
+
}
|
|
125
|
+
/** 迁移未完成时清掉本次副本,让下次启动可以重新迁移(源文件始终不动)。 */
|
|
126
|
+
async function rollback(files) {
|
|
127
|
+
for (const file of files)
|
|
128
|
+
await rm(file, { force: true }).catch(() => { });
|
|
129
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF 下载目录的纯路径计算。
|
|
3
|
+
*
|
|
4
|
+
* 优先级:显式配置 `pdfRoot` → 会话工作目录下的 `dsh-litsearch/` →
|
|
5
|
+
* 历史默认 `~/Downloads/dsh-litsearch`。保持无相对导入,便于单测直接加载。
|
|
6
|
+
*/
|
|
7
|
+
/** 工作目录(或 Downloads)下的固定子目录名。 */
|
|
8
|
+
export declare const PDF_SUBDIR = "dsh-litsearch";
|
|
9
|
+
/** 展开 `~` / `~/...`;其它路径原样返回。 */
|
|
10
|
+
export declare function expandHomePath(path: string, home?: string): string;
|
|
11
|
+
export declare function resolvePdfRootPath(configured: string, workspace: string | undefined, home?: string): string;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF 下载目录的纯路径计算。
|
|
3
|
+
*
|
|
4
|
+
* 优先级:显式配置 `pdfRoot` → 会话工作目录下的 `dsh-litsearch/` →
|
|
5
|
+
* 历史默认 `~/Downloads/dsh-litsearch`。保持无相对导入,便于单测直接加载。
|
|
6
|
+
*/
|
|
7
|
+
import { homedir } from 'node:os';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
/** 工作目录(或 Downloads)下的固定子目录名。 */
|
|
10
|
+
export const PDF_SUBDIR = 'dsh-litsearch';
|
|
11
|
+
/** 展开 `~` / `~/...`;其它路径原样返回。 */
|
|
12
|
+
export function expandHomePath(path, home = homedir()) {
|
|
13
|
+
if (path === '~')
|
|
14
|
+
return home;
|
|
15
|
+
if (path.startsWith('~/'))
|
|
16
|
+
return join(home, path.slice(2));
|
|
17
|
+
return path;
|
|
18
|
+
}
|
|
19
|
+
export function resolvePdfRootPath(configured, workspace, home = homedir()) {
|
|
20
|
+
const custom = configured.trim();
|
|
21
|
+
if (custom !== '')
|
|
22
|
+
return expandHomePath(custom, home);
|
|
23
|
+
const base = workspace?.trim() ?? '';
|
|
24
|
+
if (base !== '')
|
|
25
|
+
return join(base, PDF_SUBDIR);
|
|
26
|
+
return join(home, 'Downloads', PDF_SUBDIR);
|
|
27
|
+
}
|
package/lib/host/pdf/store.d.ts
CHANGED
|
@@ -18,8 +18,8 @@ export interface PdfResolution {
|
|
|
18
18
|
url?: string;
|
|
19
19
|
error?: string;
|
|
20
20
|
}
|
|
21
|
-
/**
|
|
22
|
-
export declare function resolvePdfRoot(): string;
|
|
21
|
+
/** 设置里的根目录;未配置时落在会话工作目录下的 dsh-litsearch/。 */
|
|
22
|
+
export declare function resolvePdfRoot(workspace?: string): string;
|
|
23
23
|
export declare function pdfCacheDir(storageRoot: string): string;
|
|
24
24
|
/** 尝试所有候选并返回第一个合法 PDF;失败时给出可展示的状态与原因。 */
|
|
25
25
|
export declare function resolvePdf(paper: Paper, signal?: AbortSignal): Promise<PdfResolution>;
|
|
@@ -31,7 +31,7 @@ export interface FetchPdfOptions {
|
|
|
31
31
|
/** 下载并校验单个 PDF;内部处理重定向,每一步都重新校验目标地址。 */
|
|
32
32
|
export declare function fetchPdf(url: string, options?: FetchPdfOptions): Promise<Buffer>;
|
|
33
33
|
/** 写入去重缓存与用户目录;返回两份路径。 */
|
|
34
|
-
export declare function persistPdf(buffer: Buffer, storageRoot: string, paper: Paper): Promise<{
|
|
34
|
+
export declare function persistPdf(buffer: Buffer, storageRoot: string, paper: Paper, workspace?: string): Promise<{
|
|
35
35
|
cachePath: string;
|
|
36
36
|
filePath: string;
|
|
37
37
|
filename: string;
|
|
@@ -41,14 +41,22 @@ export declare function readSavedPdf(filePath: string): Promise<Buffer | null>;
|
|
|
41
41
|
/** `<第一作者><年份>_<短标题>_<PMID|DOI>.pdf`,去掉路径分隔符。 */
|
|
42
42
|
export declare function readableFilename(paper: Paper): string;
|
|
43
43
|
/** 预览时读取已保存的可读文件;没有则返回 null(由调用方决定是否联网)。 */
|
|
44
|
-
export declare function readLocalPdf(paper: Paper): Promise<Buffer | null>;
|
|
44
|
+
export declare function readLocalPdf(paper: Paper, workspace?: string): Promise<Buffer | null>;
|
|
45
45
|
/** 清理 PDF 缓存目录。 */
|
|
46
46
|
export declare function cleanupPdfCache(storageRoot: string): Promise<{
|
|
47
47
|
removed: number;
|
|
48
48
|
freedBytes: number;
|
|
49
49
|
}>;
|
|
50
|
+
/**
|
|
51
|
+
* 按总量上限修剪去重缓存:超出时从最旧(mtime)开始删,直到不超过上限。
|
|
52
|
+
* `maxBytes <= 0` 表示关闭自动清理。
|
|
53
|
+
*/
|
|
54
|
+
export declare function prunePdfCache(storageRoot: string, maxBytes: number): Promise<{
|
|
55
|
+
removed: number;
|
|
56
|
+
freedBytes: number;
|
|
57
|
+
}>;
|
|
50
58
|
/** 测试下载目录是否可写(设置页)。 */
|
|
51
|
-
export declare function probePdfRoot(): Promise<{
|
|
59
|
+
export declare function probePdfRoot(workspace?: string): Promise<{
|
|
52
60
|
root: string;
|
|
53
61
|
writable: boolean;
|
|
54
62
|
error?: string;
|
package/lib/host/pdf/store.js
CHANGED
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
import { createHash } from 'node:crypto';
|
|
8
8
|
import { lookup } from 'node:dns/promises';
|
|
9
9
|
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
|
|
10
|
-
import { homedir } from 'node:os';
|
|
11
10
|
import { isIP } from 'node:net';
|
|
12
11
|
import { join } from 'node:path';
|
|
13
12
|
import { getSettings } from '../../search/config.js';
|
|
14
13
|
import { epmcXmlUrl, pdfCandidates, unpaywallPdfCandidates } from '../../search/oa.js';
|
|
14
|
+
import { resolvePdfRootPath } from './root.js';
|
|
15
15
|
const MAX_REDIRECTS = 5;
|
|
16
16
|
const FETCH_TIMEOUT_MS = 20_000;
|
|
17
17
|
const MAX_PDF_BYTES_DEFAULT = 30 * 1024 * 1024;
|
|
@@ -26,16 +26,9 @@ export class PdfFetchError extends Error {
|
|
|
26
26
|
this.status = status;
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
-
/**
|
|
30
|
-
export function resolvePdfRoot() {
|
|
31
|
-
|
|
32
|
-
if (configured === '')
|
|
33
|
-
return join(homedir(), 'Downloads', 'dsh-litsearch');
|
|
34
|
-
if (configured === '~')
|
|
35
|
-
return homedir();
|
|
36
|
-
if (configured.startsWith('~/'))
|
|
37
|
-
return join(homedir(), configured.slice(2));
|
|
38
|
-
return configured;
|
|
29
|
+
/** 设置里的根目录;未配置时落在会话工作目录下的 dsh-litsearch/。 */
|
|
30
|
+
export function resolvePdfRoot(workspace) {
|
|
31
|
+
return resolvePdfRootPath(getSettings().pdfRoot, workspace);
|
|
39
32
|
}
|
|
40
33
|
export function pdfCacheDir(storageRoot) {
|
|
41
34
|
return join(storageRoot, 'pdfs');
|
|
@@ -138,13 +131,13 @@ export async function fetchPdf(url, options = {}) {
|
|
|
138
131
|
throw new PdfFetchError('failed', `重定向超过 ${MAX_REDIRECTS} 次`);
|
|
139
132
|
}
|
|
140
133
|
/** 写入去重缓存与用户目录;返回两份路径。 */
|
|
141
|
-
export async function persistPdf(buffer, storageRoot, paper) {
|
|
134
|
+
export async function persistPdf(buffer, storageRoot, paper, workspace) {
|
|
142
135
|
const hash = createHash('sha256').update(buffer).digest('hex');
|
|
143
136
|
const cacheDir = pdfCacheDir(storageRoot);
|
|
144
137
|
await mkdir(cacheDir, { recursive: true });
|
|
145
138
|
const cachePath = join(cacheDir, `${hash}.pdf`);
|
|
146
139
|
await writeFile(cachePath, buffer);
|
|
147
|
-
const root = resolvePdfRoot();
|
|
140
|
+
const root = resolvePdfRoot(workspace);
|
|
148
141
|
await mkdir(root, { recursive: true });
|
|
149
142
|
const filename = readableFilename(paper);
|
|
150
143
|
const filePath = join(root, filename);
|
|
@@ -178,8 +171,8 @@ export function readableFilename(paper) {
|
|
|
178
171
|
return `${base.slice(0, 120)}.pdf`;
|
|
179
172
|
}
|
|
180
173
|
/** 预览时读取已保存的可读文件;没有则返回 null(由调用方决定是否联网)。 */
|
|
181
|
-
export async function readLocalPdf(paper) {
|
|
182
|
-
return readSavedPdf(join(resolvePdfRoot(), readableFilename(paper)));
|
|
174
|
+
export async function readLocalPdf(paper, workspace) {
|
|
175
|
+
return readSavedPdf(join(resolvePdfRoot(workspace), readableFilename(paper)));
|
|
183
176
|
}
|
|
184
177
|
/** 清理 PDF 缓存目录。 */
|
|
185
178
|
export async function cleanupPdfCache(storageRoot) {
|
|
@@ -206,9 +199,61 @@ export async function cleanupPdfCache(storageRoot) {
|
|
|
206
199
|
}
|
|
207
200
|
return { removed, freedBytes };
|
|
208
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* 按总量上限修剪去重缓存:超出时从最旧(mtime)开始删,直到不超过上限。
|
|
204
|
+
* `maxBytes <= 0` 表示关闭自动清理。
|
|
205
|
+
*/
|
|
206
|
+
export async function prunePdfCache(storageRoot, maxBytes) {
|
|
207
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0)
|
|
208
|
+
return { removed: 0, freedBytes: 0 };
|
|
209
|
+
const dir = pdfCacheDir(storageRoot);
|
|
210
|
+
let names;
|
|
211
|
+
try {
|
|
212
|
+
names = await readdir(dir);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return { removed: 0, freedBytes: 0 };
|
|
216
|
+
}
|
|
217
|
+
const files = [];
|
|
218
|
+
let total = 0;
|
|
219
|
+
for (const name of names) {
|
|
220
|
+
if (!name.endsWith('.pdf'))
|
|
221
|
+
continue;
|
|
222
|
+
const path = join(dir, name);
|
|
223
|
+
try {
|
|
224
|
+
const info = await stat(path);
|
|
225
|
+
if (!info.isFile())
|
|
226
|
+
continue;
|
|
227
|
+
files.push({ path, size: info.size, mtimeMs: info.mtimeMs });
|
|
228
|
+
total += info.size;
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
// 单个缓存文件读不到元信息时跳过,不影响修剪其余文件。
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
if (total <= maxBytes)
|
|
235
|
+
return { removed: 0, freedBytes: 0 };
|
|
236
|
+
files.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
|
237
|
+
let removed = 0;
|
|
238
|
+
let freedBytes = 0;
|
|
239
|
+
for (const file of files) {
|
|
240
|
+
if (total <= maxBytes)
|
|
241
|
+
break;
|
|
242
|
+
try {
|
|
243
|
+
await rm(file.path, { force: true });
|
|
244
|
+
total -= file.size;
|
|
245
|
+
freedBytes += file.size;
|
|
246
|
+
removed += 1;
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
// 删除失败继续尝试更近的文件。
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return { removed, freedBytes };
|
|
253
|
+
}
|
|
209
254
|
/** 测试下载目录是否可写(设置页)。 */
|
|
210
|
-
export async function probePdfRoot() {
|
|
211
|
-
const root = resolvePdfRoot();
|
|
255
|
+
export async function probePdfRoot(workspace) {
|
|
256
|
+
const root = resolvePdfRoot(workspace);
|
|
212
257
|
const probe = join(root, `.write-test-${process.pid}-${Date.now()}`);
|
|
213
258
|
try {
|
|
214
259
|
await mkdir(root, { recursive: true });
|
package/lib/host/router.js
CHANGED
|
@@ -6,15 +6,26 @@
|
|
|
6
6
|
* `sec-fetch-site: cross-site`。写端点额外要求自定义头 + JSON content-type,
|
|
7
7
|
* 两者一起让浏览器必须预检,堵掉跨站表单与简单请求。
|
|
8
8
|
*/
|
|
9
|
-
import { MAX_LIMIT, ROUTE_PREFIX, WRITE_HEADER } from '../search/config.js';
|
|
9
|
+
import { MAX_LIMIT, ROUTE_PREFIX, SESSION_HEADER, WRITE_HEADER } from '../search/config.js';
|
|
10
10
|
import { SearchError } from '../search/errors.js';
|
|
11
11
|
import { asRecord } from '../search/normalize.js';
|
|
12
12
|
import { isSourceId } from '../search/providers/index.js';
|
|
13
|
+
import { TASK_LANGUAGES } from '../search/types.js';
|
|
14
|
+
import { resolveSessionWorkspace } from './workspace.js';
|
|
13
15
|
const MAX_BODY_BYTES = 1_000_000;
|
|
14
16
|
const SORT_MODES = ['relevance', 'citations', 'year'];
|
|
15
|
-
const LANGUAGES =
|
|
17
|
+
const LANGUAGES = TASK_LANGUAGES;
|
|
16
18
|
const ORIGINS = ['model', 'panel', 'history', 'api'];
|
|
17
19
|
const SCREENINGS = ['unread', 'read', 'included', 'excluded'];
|
|
20
|
+
const PDF_STATUSES = [
|
|
21
|
+
'unknown',
|
|
22
|
+
'available',
|
|
23
|
+
'downloaded',
|
|
24
|
+
'html-only',
|
|
25
|
+
'paywalled',
|
|
26
|
+
'blocked',
|
|
27
|
+
'failed',
|
|
28
|
+
];
|
|
18
29
|
const EXPORT_FORMATS = ['csv', 'ris', 'bibtex', 'md'];
|
|
19
30
|
const CLEANUP_SCOPES = ['tasks', 'pdf', 'all'];
|
|
20
31
|
const MAX_ANNOTATIONS = 50;
|
|
@@ -30,6 +41,13 @@ function isTrusted(req) {
|
|
|
30
41
|
return false;
|
|
31
42
|
return isLoopbackHost(req.headers.host);
|
|
32
43
|
}
|
|
44
|
+
/** 面板把当前会话 id 放在请求头里;iframe 预览等无法带头部的 GET 用查询参数。 */
|
|
45
|
+
function readSessionId(req, params) {
|
|
46
|
+
const header = req.headers[SESSION_HEADER];
|
|
47
|
+
const raw = typeof header === 'string' && header.trim() !== '' ? header : params.get('session') ?? '';
|
|
48
|
+
const id = raw.trim();
|
|
49
|
+
return id === '' ? undefined : id;
|
|
50
|
+
}
|
|
33
51
|
/** 写端点要求自定义头 + JSON:跨站请求无法在不触发预检的情况下伪造。 */
|
|
34
52
|
function hasWriteIntent(req) {
|
|
35
53
|
if (req.headers[WRITE_HEADER] !== '1')
|
|
@@ -521,6 +539,9 @@ function readResultsFilter(params) {
|
|
|
521
539
|
const missingFullText = params.get('missingFullText');
|
|
522
540
|
if (missingFullText === '1' || missingFullText === 'true')
|
|
523
541
|
filter.missingFullText = true;
|
|
542
|
+
const pdfStatus = readEnum(params.get('pdfStatus') ?? undefined, 'pdfStatus', PDF_STATUSES);
|
|
543
|
+
if (pdfStatus !== undefined)
|
|
544
|
+
filter.pdfStatus = pdfStatus;
|
|
524
545
|
const groupId = params.get('groupId')?.trim() ?? '';
|
|
525
546
|
if (groupId !== '')
|
|
526
547
|
filter.groupId = groupId;
|
|
@@ -549,7 +570,7 @@ function parseSegments(pathname) {
|
|
|
549
570
|
}
|
|
550
571
|
});
|
|
551
572
|
}
|
|
552
|
-
async function handleTasks(service, req, res, seg, params) {
|
|
573
|
+
async function handleTasks(service, req, res, seg, params, sessionId) {
|
|
553
574
|
const id = seg[1];
|
|
554
575
|
if (id === undefined) {
|
|
555
576
|
if (req.method === 'GET') {
|
|
@@ -558,7 +579,8 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
558
579
|
}
|
|
559
580
|
if (req.method === 'POST') {
|
|
560
581
|
const input = readCreateInput(await readJsonBody(req));
|
|
561
|
-
const
|
|
582
|
+
const workspace = await resolveSessionWorkspace(sessionId);
|
|
583
|
+
const task = await service.create({ ...input, origin: 'panel', workspace });
|
|
562
584
|
ok(res, await service.getDetail(task.id), 201);
|
|
563
585
|
return;
|
|
564
586
|
}
|
|
@@ -617,7 +639,7 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
617
639
|
fail(res, 405, 'bad_request', 'activate 仅支持 POST');
|
|
618
640
|
return;
|
|
619
641
|
}
|
|
620
|
-
const task = await service.activate(id);
|
|
642
|
+
const task = await service.activate(id, await resolveSessionWorkspace(sessionId));
|
|
621
643
|
ok(res, await service.getDetail(task.id));
|
|
622
644
|
return;
|
|
623
645
|
}
|
|
@@ -658,7 +680,7 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
658
680
|
fail(res, 405, 'bad_request', 'download 仅支持 POST');
|
|
659
681
|
return;
|
|
660
682
|
}
|
|
661
|
-
ok(res, await service.downloadPdf(id, paperId));
|
|
683
|
+
ok(res, await service.downloadPdf(id, paperId, await resolveSessionWorkspace(sessionId)));
|
|
662
684
|
return;
|
|
663
685
|
}
|
|
664
686
|
if (op === 'pdf') {
|
|
@@ -667,7 +689,7 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
667
689
|
return;
|
|
668
690
|
}
|
|
669
691
|
try {
|
|
670
|
-
const file = await service.previewPdf(id, paperId);
|
|
692
|
+
const file = await service.previewPdf(id, paperId, await resolveSessionWorkspace(sessionId));
|
|
671
693
|
sendPdf(req, res, file.buffer, file.filename);
|
|
672
694
|
}
|
|
673
695
|
catch (error) {
|
|
@@ -699,7 +721,7 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
699
721
|
}
|
|
700
722
|
const body = await readJsonBody(req);
|
|
701
723
|
const ids = readStringArray(body['paperIds'], 'paperIds', 50, 80);
|
|
702
|
-
ok(res, await service.zoteroSync(id, ids === undefined || ids === null ? undefined : ids));
|
|
724
|
+
ok(res, await service.zoteroSync(id, ids === undefined || ids === null ? undefined : ids, await resolveSessionWorkspace(sessionId)));
|
|
703
725
|
return;
|
|
704
726
|
}
|
|
705
727
|
if (action === 'compare') {
|
|
@@ -725,7 +747,8 @@ async function handleTasks(service, req, res, seg, params) {
|
|
|
725
747
|
throw new SearchError('conflict', '任务正在运行');
|
|
726
748
|
// 面板里主动重跑 = 用户正在这个任务里工作,把它设为「当前任务」。
|
|
727
749
|
service.setCurrentTask(id);
|
|
728
|
-
|
|
750
|
+
const workspace = await resolveSessionWorkspace(sessionId);
|
|
751
|
+
void service.run(id, mode, 'panel', workspace).catch((error) => {
|
|
729
752
|
console.warn('[dsh-plugin-litsearch] 任务执行失败:', error);
|
|
730
753
|
});
|
|
731
754
|
ok(res, await service.getDetail(id), 202);
|
|
@@ -798,14 +821,15 @@ export function createHandler(service) {
|
|
|
798
821
|
return;
|
|
799
822
|
}
|
|
800
823
|
if (seg[1] === 'storage' && req.method === 'POST') {
|
|
801
|
-
|
|
824
|
+
const workspace = await resolveSessionWorkspace(readSessionId(req, url.searchParams));
|
|
825
|
+
ok(res, await service.storageProbe(workspace));
|
|
802
826
|
return;
|
|
803
827
|
}
|
|
804
828
|
fail(res, 404, 'not_found', `未知设置端点 ${seg[1] ?? ''}`);
|
|
805
829
|
return;
|
|
806
830
|
}
|
|
807
831
|
if (head === 'tasks') {
|
|
808
|
-
await handleTasks(service, req, res, seg, url.searchParams);
|
|
832
|
+
await handleTasks(service, req, res, seg, url.searchParams, readSessionId(req, url.searchParams));
|
|
809
833
|
return;
|
|
810
834
|
}
|
|
811
835
|
if (head === 'export') {
|
|
@@ -20,6 +20,8 @@ export interface PaperFilter {
|
|
|
20
20
|
hasPdf?: boolean;
|
|
21
21
|
/** 只保留尚无可用全文(PDF 或 EPMC HTML)的论文。 */
|
|
22
22
|
missingFullText?: boolean;
|
|
23
|
+
/** 只保留有效全文状态恰为该值的论文(与覆盖率的 pdfStatusOf 同一口径)。 */
|
|
24
|
+
pdfStatus?: PdfStatus;
|
|
23
25
|
/** 只保留归入该分组的论文。 */
|
|
24
26
|
groupId?: string;
|
|
25
27
|
/** 期刊/出处(精确匹配,不分大小写)。 */
|
|
@@ -35,6 +37,8 @@ export declare function filterPapers(task: SearchTask, filter?: PaperFilter): Pa
|
|
|
35
37
|
export declare function screeningOf(task: SearchTask, paper: Paper): Screening;
|
|
36
38
|
/** 有效全文状态:没有人工/下载记录时,上游给的 PDF 直链视为可用。 */
|
|
37
39
|
export declare function pdfStatusOf(task: SearchTask, paper: Paper): PdfStatus;
|
|
38
|
-
export declare function matchesLanguage(paper:
|
|
40
|
+
export declare function matchesLanguage(paper: {
|
|
41
|
+
language?: string;
|
|
42
|
+
}, language: TaskLanguage): boolean;
|
|
39
43
|
/** 稳定的筛选指纹(统计缓存 key 的一部分)。 */
|
|
40
44
|
export declare function filterKey(filter: PaperFilter): string;
|
package/lib/host/tasks/filter.js
CHANGED
|
@@ -38,6 +38,9 @@ export function filterPapers(task, filter = {}) {
|
|
|
38
38
|
if (status === 'downloaded' || status === 'available' || status === 'html-only')
|
|
39
39
|
return false;
|
|
40
40
|
}
|
|
41
|
+
if (filter.pdfStatus !== undefined && pdfStatusOf(task, paper) !== filter.pdfStatus) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
41
44
|
if (filter.groupId !== undefined && task.states[paper.paperId]?.groupId !== filter.groupId) {
|
|
42
45
|
return false;
|
|
43
46
|
}
|
|
@@ -69,11 +72,52 @@ export function pdfStatusOf(task, paper) {
|
|
|
69
72
|
export function matchesLanguage(paper, language) {
|
|
70
73
|
if (language === 'any')
|
|
71
74
|
return true;
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return
|
|
75
|
+
const raw = (paper.language ?? '').trim().toLowerCase();
|
|
76
|
+
// 兼容 `en-US` / `zh-CN` 这类带地区的写法,只取主语言码。
|
|
77
|
+
const code = LANGUAGE_ALIASES[raw] ?? LANGUAGE_ALIASES[raw.split(/[-_]/u)[0] ?? ''];
|
|
78
|
+
return code !== undefined && code === language;
|
|
76
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* 上游语言标记 → 统一的 PubMed 三字母码。
|
|
82
|
+
*
|
|
83
|
+
* PubMed / Europe PMC 给三字母码,OpenAlex 等给两字母码,这里都收敛到同一套,
|
|
84
|
+
* 避免语言筛选把同一种语言判成两种。
|
|
85
|
+
*/
|
|
86
|
+
const LANGUAGE_ALIASES = {
|
|
87
|
+
eng: 'eng',
|
|
88
|
+
en: 'eng',
|
|
89
|
+
english: 'eng',
|
|
90
|
+
chi: 'chi',
|
|
91
|
+
zho: 'chi',
|
|
92
|
+
zh: 'chi',
|
|
93
|
+
chinese: 'chi',
|
|
94
|
+
jpn: 'jpn',
|
|
95
|
+
ja: 'jpn',
|
|
96
|
+
japanese: 'jpn',
|
|
97
|
+
kor: 'kor',
|
|
98
|
+
ko: 'kor',
|
|
99
|
+
korean: 'kor',
|
|
100
|
+
fre: 'fre',
|
|
101
|
+
fra: 'fre',
|
|
102
|
+
fr: 'fre',
|
|
103
|
+
french: 'fre',
|
|
104
|
+
ger: 'ger',
|
|
105
|
+
deu: 'ger',
|
|
106
|
+
de: 'ger',
|
|
107
|
+
german: 'ger',
|
|
108
|
+
spa: 'spa',
|
|
109
|
+
es: 'spa',
|
|
110
|
+
spanish: 'spa',
|
|
111
|
+
rus: 'rus',
|
|
112
|
+
ru: 'rus',
|
|
113
|
+
russian: 'rus',
|
|
114
|
+
ita: 'ita',
|
|
115
|
+
it: 'ita',
|
|
116
|
+
italian: 'ita',
|
|
117
|
+
por: 'por',
|
|
118
|
+
pt: 'por',
|
|
119
|
+
portuguese: 'por',
|
|
120
|
+
};
|
|
77
121
|
/** 稳定的筛选指纹(统计缓存 key 的一部分)。 */
|
|
78
122
|
export function filterKey(filter) {
|
|
79
123
|
return [
|
|
@@ -88,6 +132,7 @@ export function filterKey(filter) {
|
|
|
88
132
|
filter.language ?? '',
|
|
89
133
|
filter.hasPdf === true ? '1' : '',
|
|
90
134
|
filter.missingFullText === true ? '1' : '',
|
|
135
|
+
filter.pdfStatus ?? '',
|
|
91
136
|
filter.groupId ?? '',
|
|
92
137
|
filter.venue ?? '',
|
|
93
138
|
filter.author ?? '',
|