dsh-plugin-litsearch 0.0.0-stage → 0.2.0

package/lib/host/pdf/store.d.ts

@@ -0,0 +1,57 @@
+/**
+ * PDF 获取与落地。
+ *
+ * 安全约束：只允许 http/https、拒绝回环/私网/链路本地地址、最多 5 次重定向、
+ * 20 秒超时、体积上限可配；只有以 `%PDF-` 开头的内容才会被当成 PDF。
+ */
+import type { Paper, PdfStatus } from '../../search/types.js';
+export type PdfFailure = 'paywalled' | 'blocked' | 'failed' | 'not_pdf' | 'too_large' | 'unsafe';
+export declare class PdfFetchError extends Error {
+    readonly kind: PdfFailure;
+    readonly status?: number;
+    constructor(kind: PdfFailure, message: string, status?: number);
+}
+export interface PdfResolution {
+    ok: boolean;
+    status: PdfStatus;
+    buffer?: Buffer;
+    url?: string;
+    error?: string;
+}
+/** 设置里的根目录；空串回落到 ~/Downloads/dsh-litsearch。 */
+export declare function resolvePdfRoot(): string;
+export declare function pdfCacheDir(storageRoot: string): string;
+/** 尝试所有候选并返回第一个合法 PDF；失败时给出可展示的状态与原因。 */
+export declare function resolvePdf(paper: Paper, signal?: AbortSignal): Promise<PdfResolution>;
+export interface FetchPdfOptions {
+    maxBytes?: number;
+    signal?: AbortSignal;
+    timeoutMs?: number;
+}
+/** 下载并校验单个 PDF；内部处理重定向，每一步都重新校验目标地址。 */
+export declare function fetchPdf(url: string, options?: FetchPdfOptions): Promise<Buffer>;
+/** 写入去重缓存与用户目录；返回两份路径。 */
+export declare function persistPdf(buffer: Buffer, storageRoot: string, paper: Paper): Promise<{
+    cachePath: string;
+    filePath: string;
+    filename: string;
+}>;
+/** 已下载文件仍存在时直接读取，避免重复下载。 */
+export declare function readSavedPdf(filePath: string): Promise<Buffer | null>;
+/** `<第一作者><年份>_<短标题>_<PMID|DOI>.pdf`，去掉路径分隔符。 */
+export declare function readableFilename(paper: Paper): string;
+/** 预览时读取已保存的可读文件；没有则返回 null（由调用方决定是否联网）。 */
+export declare function readLocalPdf(paper: Paper): Promise<Buffer | null>;
+/** 清理 PDF 缓存目录。 */
+export declare function cleanupPdfCache(storageRoot: string): Promise<{
+    removed: number;
+    freedBytes: number;
+}>;
+/** 测试下载目录是否可写（设置页）。 */
+export declare function probePdfRoot(): Promise<{
+    root: string;
+    writable: boolean;
+    error?: string;
+}>;
+/** 供路由/服务判断是否有 EPMC HTML 全文。 */
+export declare function epmcFullTextAvailable(paper: Paper): boolean;

package/lib/host/pdf/store.js

@@ -0,0 +1,291 @@
+/**
+ * PDF 获取与落地。
+ *
+ * 安全约束：只允许 http/https、拒绝回环/私网/链路本地地址、最多 5 次重定向、
+ * 20 秒超时、体积上限可配；只有以 `%PDF-` 开头的内容才会被当成 PDF。
+ */
+import { createHash } from 'node:crypto';
+import { lookup } from 'node:dns/promises';
+import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { isIP } from 'node:net';
+import { join } from 'node:path';
+import { getSettings } from '../../search/config.js';
+import { epmcXmlUrl, pdfCandidates, unpaywallPdfCandidates } from '../../search/oa.js';
+const MAX_REDIRECTS = 5;
+const FETCH_TIMEOUT_MS = 20_000;
+const MAX_PDF_BYTES_DEFAULT = 30 * 1024 * 1024;
+export class PdfFetchError extends Error {
+    kind;
+    status;
+    constructor(kind, message, status) {
+        super(message);
+        this.name = 'PdfFetchError';
+        this.kind = kind;
+        if (status !== undefined)
+            this.status = status;
+    }
+}
+/** 设置里的根目录；空串回落到 ~/Downloads/dsh-litsearch。 */
+export function resolvePdfRoot() {
+    const configured = getSettings().pdfRoot.trim();
+    if (configured === '')
+        return join(homedir(), 'Downloads', 'dsh-litsearch');
+    if (configured === '~')
+        return homedir();
+    if (configured.startsWith('~/'))
+        return join(homedir(), configured.slice(2));
+    return configured;
+}
+export function pdfCacheDir(storageRoot) {
+    return join(storageRoot, 'pdfs');
+}
+/** 尝试所有候选并返回第一个合法 PDF；失败时给出可展示的状态与原因。 */
+export async function resolvePdf(paper, signal) {
+    const settings = getSettings();
+    const maxBytes = settings.pdfMaxMb * 1024 * 1024;
+    const candidates = pdfCandidates(paper);
+    const extra = await unpaywallPdfCandidates(paper.doi, settings.contactEmail, Math.min(settings.requestTimeoutMs, 8_000), signal);
+    for (const url of extra)
+        if (!candidates.includes(url))
+            candidates.push(url);
+    let last = null;
+    let sawBlocked = false;
+    let sawPaywalled = false;
+    for (const url of candidates) {
+        try {
+            const buffer = await fetchPdf(url, { maxBytes, signal });
+            return { ok: true, status: 'available', buffer, url };
+        }
+        catch (error) {
+            if (signal?.aborted === true)
+                throw error;
+            const failure = error instanceof PdfFetchError
+                ? error
+                : new PdfFetchError('failed', error instanceof Error ? error.message : String(error));
+            last = failure;
+            if (failure.kind === 'blocked')
+                sawBlocked = true;
+            if (failure.kind === 'paywalled')
+                sawPaywalled = true;
+        }
+    }
+    const status = sawBlocked ? 'blocked' : sawPaywalled ? 'paywalled' : 'failed';
+    const error = last === null
+        ? '没有可尝试的 PDF 地址'
+        : `${last.message}${candidates.length > 1 ? `（已尝试 ${candidates.length} 个候选）` : ''}`;
+    return { ok: false, status, error };
+}
+/** 下载并校验单个 PDF；内部处理重定向，每一步都重新校验目标地址。 */
+export async function fetchPdf(url, options = {}) {
+    const maxBytes = options.maxBytes ?? MAX_PDF_BYTES_DEFAULT;
+    const deadline = Date.now() + (options.timeoutMs ?? FETCH_TIMEOUT_MS);
+    let current = url;
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
+        await assertPublicUrl(current);
+        const controller = new AbortController();
+        const remaining = Math.max(1_000, deadline - Date.now());
+        const timer = setTimeout(() => controller.abort(), remaining);
+        const onAbort = () => controller.abort();
+        options.signal?.addEventListener('abort', onAbort);
+        try {
+            const response = await fetch(current, {
+                redirect: 'manual',
+                signal: controller.signal,
+                headers: { accept: 'application/pdf,application/octet-stream;q=0.9,*/*;q=0.5' },
+            });
+            if (response.status >= 300 && response.status < 400) {
+                const location = response.headers.get('location');
+                if (location === null)
+                    throw new PdfFetchError('failed', `重定向缺少 Location：${current}`);
+                current = new URL(location, current).toString();
+                continue;
+            }
+            if (response.status === 401 || response.status === 402) {
+                throw new PdfFetchError('paywalled', `需要订阅或登录（HTTP ${response.status}）`, response.status);
+            }
+            if (response.status === 403 || response.status === 429) {
+                throw new PdfFetchError('blocked', `出版商拒绝访问（HTTP ${response.status}）`, response.status);
+            }
+            if (!response.ok) {
+                throw new PdfFetchError('failed', `下载失败（HTTP ${response.status}）`, response.status);
+            }
+            const declared = Number(response.headers.get('content-length') ?? '0');
+            if (Number.isFinite(declared) && declared > maxBytes) {
+                throw new PdfFetchError('too_large', `文件超过 ${Math.round(maxBytes / 1024 / 1024)} MB`);
+            }
+            const buffer = await readCapped(response, maxBytes);
+            if (buffer.length < 5 || buffer.subarray(0, 5).toString('latin1') !== '%PDF-') {
+                throw new PdfFetchError('not_pdf', '返回内容不是 PDF（可能是 HTML 登录/摘要页）');
+            }
+            return buffer;
+        }
+        catch (error) {
+            if (options.signal?.aborted === true)
+                throw error;
+            if (error instanceof PdfFetchError)
+                throw error;
+            if (controller.signal.aborted) {
+                throw new PdfFetchError('failed', '下载超时');
+            }
+            throw new PdfFetchError('failed', error instanceof Error ? error.message : String(error));
+        }
+        finally {
+            clearTimeout(timer);
+            options.signal?.removeEventListener('abort', onAbort);
+        }
+    }
+    throw new PdfFetchError('failed', `重定向超过 ${MAX_REDIRECTS} 次`);
+}
+/** 写入去重缓存与用户目录；返回两份路径。 */
+export async function persistPdf(buffer, storageRoot, paper) {
+    const hash = createHash('sha256').update(buffer).digest('hex');
+    const cacheDir = pdfCacheDir(storageRoot);
+    await mkdir(cacheDir, { recursive: true });
+    const cachePath = join(cacheDir, `${hash}.pdf`);
+    await writeFile(cachePath, buffer);
+    const root = resolvePdfRoot();
+    await mkdir(root, { recursive: true });
+    const filename = readableFilename(paper);
+    const filePath = join(root, filename);
+    await writeFile(filePath, buffer);
+    return { cachePath, filePath, filename };
+}
+/** 已下载文件仍存在时直接读取，避免重复下载。 */
+export async function readSavedPdf(filePath) {
+    try {
+        return await readFile(filePath);
+    }
+    catch {
+        return null;
+    }
+}
+/** `<第一作者><年份>_<短标题>_<PMID|DOI>.pdf`，去掉路径分隔符。 */
+export function readableFilename(paper) {
+    const first = paper.authors[0]?.trim() ?? '';
+    const author = first === '' ? 'Anon' : first.split(/\s+/u).at(-1) ?? 'Anon';
+    const year = paper.year === null ? 'nd' : String(paper.year);
+    const title = paper.title
+        .normalize('NFKC')
+        .replace(/[\\/:*?"<>|\r\n]+/gu, ' ')
+        .replace(/\s+/gu, ' ')
+        .trim()
+        .slice(0, 48)
+        .trim();
+    const id = paper.ids.pmid !== '' ? `PMID${paper.ids.pmid}` : paper.doi !== '' ? `DOI${paper.doi}` : paper.paperId;
+    const safeId = id.replace(/[^A-Za-z0-9._-]+/gu, '_').slice(0, 64);
+    const base = `${author}${year}_${title === '' ? 'untitled' : title}_${safeId}`;
+    return `${base.slice(0, 120)}.pdf`;
+}
+/** 预览时读取已保存的可读文件；没有则返回 null（由调用方决定是否联网）。 */
+export async function readLocalPdf(paper) {
+    return readSavedPdf(join(resolvePdfRoot(), readableFilename(paper)));
+}
+/** 清理 PDF 缓存目录。 */
+export async function cleanupPdfCache(storageRoot) {
+    const dir = pdfCacheDir(storageRoot);
+    let names;
+    try {
+        names = await readdir(dir);
+    }
+    catch {
+        return { removed: 0, freedBytes: 0 };
+    }
+    let removed = 0;
+    let freedBytes = 0;
+    for (const name of names) {
+        const path = join(dir, name);
+        try {
+            freedBytes += (await stat(path)).size;
+            await rm(path, { force: true });
+            removed += 1;
+        }
+        catch {
+            // 单个缓存文件删除失败不影响其余文件。
+        }
+    }
+    return { removed, freedBytes };
+}
+/** 测试下载目录是否可写（设置页）。 */
+export async function probePdfRoot() {
+    const root = resolvePdfRoot();
+    const probe = join(root, `.write-test-${process.pid}-${Date.now()}`);
+    try {
+        await mkdir(root, { recursive: true });
+        await writeFile(probe, 'ok', 'utf8');
+        await rm(probe, { force: true });
+        return { root, writable: true };
+    }
+    catch (error) {
+        return {
+            root,
+            writable: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
+/** 供路由/服务判断是否有 EPMC HTML 全文。 */
+export function epmcFullTextAvailable(paper) {
+    return epmcXmlUrl(paper) !== null;
+}
+async function readCapped(response, maxBytes) {
+    if (response.body === null)
+        return Buffer.alloc(0);
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > maxBytes) {
+            await reader.cancel();
+            throw new PdfFetchError('too_large', `文件超过 ${Math.round(maxBytes / 1024 / 1024)} MB`);
+        }
+        chunks.push(value);
+    }
+    return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)));
+}
+async function assertPublicUrl(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        throw new PdfFetchError('unsafe', `非法地址：${raw}`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new PdfFetchError('unsafe', `只允许 http/https：${url.protocol}`);
+    }
+    const host = url.hostname.replace(/^\[|\]$/gu, '').toLowerCase();
+    if (host === 'localhost' || host.endsWith('.local') || host.endsWith('.internal')) {
+        throw new PdfFetchError('unsafe', `拒绝访问本机/内网地址：${host}`);
+    }
+    const addresses = isIP(host) !== 0 ? [{ address: host }] : await lookup(host, { all: true }).catch(() => []);
+    if (addresses.length === 0)
+        throw new PdfFetchError('failed', `无法解析域名：${host}`);
+    for (const entry of addresses) {
+        if (isPrivateAddress(entry.address)) {
+            throw new PdfFetchError('unsafe', `拒绝访问内网地址：${entry.address}`);
+        }
+    }
+}
+function isPrivateAddress(address) {
+    const value = address.toLowerCase();
+    if (value === '::1' || value === '0:0:0:0:0:0:0:1')
+        return true;
+    if (value.startsWith('fe80:') || value.startsWith('fc') || value.startsWith('fd'))
+        return true;
+    const parts = value.split('.').map((part) => Number(part));
+    if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part)))
+        return false;
+    const [a, b] = parts;
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 100 && b >= 64 && b <= 127) ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168));
+}

package/lib/host/router.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Host HTTP 路由：`/dsh-litsearch/api/*`。
+ *
+ * 面板与设置页都从这里取数；统一信封 `{ok:true,data}` / `{ok:false,error}`。
+ * 信任栅栏：Host 头必须是回环地址（防 DNS rebinding），并拒绝
+ * `sec-fetch-site: cross-site`。写端点额外要求自定义头 + JSON content-type，
+ * 两者一起让浏览器必须预检，堵掉跨站表单与简单请求。
+ */
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import type { TaskService } from './tasks/service.js';
+/** dsh `webServer` 的最小结构接口（类型不在已安装的 dsh 包里）。 */
+export interface WebServerLike {
+    register(options: {
+        kind: 'exact' | 'prefix';
+        path: string;
+        handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>;
+    }): () => void;
+}
+/** 构造挂在 `webServer` 上的前缀路由处理器。 */
+export declare function createHandler(service: TaskService): (req: IncomingMessage, res: ServerResponse) => Promise<void>;