@a9i5k4/dsh-literature 0.2.3 → 0.2.5

package/lib/index.js

@@ -554,6 +554,12 @@ function extractIdentifiers(text, options = {}) {
     while ((m = ISBN.exec(source)) !== null) {
       push(out, seen, { kind: "isbn", value: m[1].replace(/[-\s]/g, ""), display: m[1], index: offset + m.index, confidence: 0.85 + confidenceBonus });
     }
+    URL_RE.lastIndex = 0;
+    while ((m = URL_RE.exec(source)) !== null) {
+      const raw = trimRight(m[0]);
+      if (/doi\.org|arxiv\.org|dx\.doi/i.test(raw)) continue;
+      push(out, seen, { kind: "url", value: raw, display: raw, index: offset + m.index, confidence: 0.9 + confidenceBonus });
+    }
   };
   scan(src);
   for (const url of linkTargets) scan(url, 0, 0.02);
@@ -571,7 +577,7 @@ function extractIdentifiers(text, options = {}) {
   }
   return out.sort((a, b) => a.index - b.index);
 }
-var TRIM_RIGHT, CJK_PUNCT, DOI_TAIL, DOI_CORE, DOI_HINT, DOI_LABEL, ARXIV_NEW, ARXIV_OLD, ARXIV_URL, PMID, ISBN, QUOTED, QUOTED_FALLBACK;
+var TRIM_RIGHT, CJK_PUNCT, DOI_TAIL, DOI_CORE, DOI_HINT, DOI_LABEL, ARXIV_NEW, ARXIV_OLD, URL_RE, ARXIV_URL, PMID, ISBN, QUOTED, QUOTED_FALLBACK;
 var init_identifiers = __esm({
   "src/node/extract/identifiers.js"() {
     TRIM_RIGHT = /[.,;:!?。、，；：！？…—>'">）》\]]+$/;
@@ -582,6 +588,7 @@ var init_identifiers = __esm({
     DOI_LABEL = new RegExp(`\\bDOI\\s*[:\uFF1A]\\s*(10\\.\\d{4,9}\\/${DOI_TAIL})`, "gi");
     ARXIV_NEW = /\barXiv\s*[:. ]?\s*(\d{4}\.\d{4,5})(v\d+)?\b/gi;
     ARXIV_OLD = /\barXiv\s*[:. ]?\s*([a-z][a-z-]*(?:\.[A-Z]{2})?\/\d{7})(v\d+)?\b/gi;
+    URL_RE = /\bhttps?:\/\/[^\s<>"']+/gi;
     ARXIV_URL = /arxiv\.org\/(?:abs|pdf)\/([^\s"'?#>]+?)(?:\.pdf)?(?=[\s"'?#>()]|$)/gi;
     PMID = /\bPMID\s*[:：]?\s*(\d{1,8})\b/gi;
     ISBN = /\bISBN(?:-1[03])?\s*[:：]?\s*((?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx])\b/gi;
@@ -1856,6 +1863,136 @@ var init_local_api = __esm({
   }
 });
 
+// src/node/metadata/search.js
+var search_exports = {};
+__export(search_exports, {
+  searchCandidates: () => searchCandidates
+});
+function crossrefType(t) {
+  switch (t) {
+    case "journal-article":
+      return "journalArticle";
+    case "book":
+      return "book";
+    case "book-chapter":
+      return "bookSection";
+    case "proceedings-article":
+      return "conferencePaper";
+    case "posted-content":
+      return "preprint";
+    case "dissertation":
+      return "thesis";
+    case "report":
+      return "report";
+    case "webpage":
+      return "webpage";
+    default:
+      return "journalArticle";
+  }
+}
+async function searchCandidates(query, { rows = 8 } = {}) {
+  const q = String(query ?? "").trim();
+  if (!q) return [];
+  const url = `https://api.crossref.org/works?query.bibliographic=${encodeURIComponent(q)}&rows=${Math.max(1, Math.min(rows, 20))}&select=DOI,title,author,issued,container-title,type,volume,issue,page`;
+  let data;
+  try {
+    data = await httpGetJson(url);
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const it of data?.message?.items ?? []) {
+    const title = it.title?.[0];
+    if (!title) continue;
+    out.push({
+      doi: it.DOI ?? "",
+      title,
+      authors: (it.author ?? []).map((a) => ({ firstName: a.given ?? "", lastName: a.family ?? "" })),
+      year: it.issued?.["date-parts"]?.[0]?.[0] ?? null,
+      container: it["container-title"]?.[0] ?? "",
+      volume: it.volume ?? "",
+      issue: it.issue ?? "",
+      pages: it.page ?? "",
+      itemType: crossrefType(it.type)
+    });
+  }
+  return out;
+}
+var init_search = __esm({
+  "src/node/metadata/search.js"() {
+    init_net();
+  }
+});
+
+// src/node/metadata/url.js
+var url_exports = {};
+__export(url_exports, {
+  resolveUrlPage: () => resolveUrlPage
+});
+function firstMatch(html, patterns) {
+  for (const re of patterns) {
+    const m = re.exec(html);
+    if (m?.[1]) return m[1].trim();
+  }
+  return "";
+}
+async function resolveUrlPage(url, { timeoutMs = 15e3 } = {}) {
+  let html = "";
+  let error2 = "";
+  try {
+    const res = await httpGet(url, {
+      timeoutMs,
+      accept: "text/html,application/xhtml+xml,*/*;q=0.8",
+      // Many publisher pages 403 bare requests; a browser UA keeps them open.
+      headers: { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" }
+    });
+    html = await res.text();
+  } catch (e) {
+    error2 = e?.message ?? String(e);
+  }
+  const title = firstMatch(html, META_PATTERNS);
+  const doi = firstMatch(html, DOI_PATTERNS);
+  const author = firstMatch(html, AUTHOR_PATTERNS);
+  const year = firstMatch(html, DATE_PATTERNS);
+  const cleanedTitle = title.replace(/[|–-].*$/, "").trim();
+  return {
+    title: cleanedTitle || "",
+    doi,
+    authors: author ? [{ firstName: "", lastName: author }] : [],
+    year: year ? Number(year) : null,
+    url,
+    error: error2
+  };
+}
+var META_PATTERNS, DOI_PATTERNS, AUTHOR_PATTERNS, DATE_PATTERNS;
+var init_url = __esm({
+  "src/node/metadata/url.js"() {
+    init_net();
+    META_PATTERNS = [
+      // <meta name="citation_title" content="..."> — order of attrs varies
+      /<meta[^>]+name=["']citation_title["'][^>]+content=["']([^"']+)["']/i,
+      /<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_title["']/i,
+      // Open Graph
+      /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i,
+      /<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["']/i,
+      // fallback
+      /<title[^>]*>([^<]+)<\/title>/i
+    ];
+    DOI_PATTERNS = [
+      /<meta[^>]+name=["']citation_doi["'][^>]+content=["']([^"']+)["']/i,
+      /<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_doi["']/i
+    ];
+    AUTHOR_PATTERNS = [
+      /<meta[^>]+name=["']citation_author["'][^>]+content=["']([^"']+)["']/i,
+      /<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_author["']/i
+    ];
+    DATE_PATTERNS = [
+      /<meta[^>]+name=["']citation_publication_date["'][^>]+content=["'](\d{4})/i,
+      /<meta[^>]+content=["'](\d{4})["'][^>]+name=["']citation_publication_date["']/i
+    ];
+  }
+});
+
 // src/node/importer.js
 var importer_exports = {};
 __export(importer_exports, {
@@ -1935,13 +2072,23 @@ async function importDir(dir, { autoResolve = true } = {}) {
         skipped.push({ file: name2, reason: "duplicate" });
         continue;
       }
+      let pdf = null;
+      try {
+        const content = await readFile3(abs);
+        if (content.subarray(0, 5).equals(PDF_MAGIC2)) {
+          pdf = await savePdfBuffer(provisional.key, content);
+        }
+      } catch (e) {
+        warn(`copy pdf failed for ${name2}:`, e.message);
+      }
       item = await putItem({
         ...provisional,
         kind: hit.kind,
         rawValue: hit.value,
         display: hit.value,
         title: provisional.title || hit.value,
-        state: "discovered",
+        state: pdf ? "fetched" : "discovered",
+        pdf,
         sourceFile: abs,
         createdAt: Date.now()
       });
@@ -2083,6 +2230,7 @@ var init_importer = __esm({
 // src/node/pipeline.js
 var pipeline_exports = {};
 __export(pipeline_exports, {
+  addCandidate: () => addCandidate,
   arxivBase: () => arxivBase,
   discardItem: () => discardItem,
   fetchItemPdf: () => fetchItemPdf,
@@ -2095,6 +2243,7 @@ __export(pipeline_exports, {
   retryItem: () => retryItem,
   saveItem: () => saveItem,
   scanText: () => scanText,
+  searchCandidates: () => searchCandidates2,
   startTask: () => startTask
 });
 import { randomUUID as randomUUID2 } from "node:crypto";
@@ -2141,14 +2290,21 @@ async function scanText(text) {
   const existing = await listItems();
   const created = [];
   for (const hit of found) {
-    const provisional = buildItem({
-      doi: hit.kind === "doi" ? hit.value : "",
-      arxiv: hit.kind === "arxiv" ? hit.value : "",
-      isbn: hit.kind === "isbn" ? hit.value : "",
-      pmid: hit.kind === "pmid" ? hit.value : "",
-      title: hit.kind === "title" ? hit.value : ""
-    });
-    if (!provisional.key) continue;
+    let provisional;
+    if (hit.kind === "url") {
+      provisional = buildItem({ title: hit.value });
+      if (!provisional.key) continue;
+      provisional = { ...provisional, title: hit.value };
+    } else {
+      provisional = buildItem({
+        doi: hit.kind === "doi" ? hit.value : "",
+        arxiv: hit.kind === "arxiv" ? hit.value : "",
+        isbn: hit.kind === "isbn" ? hit.value : "",
+        pmid: hit.kind === "pmid" ? hit.value : "",
+        title: hit.kind === "title" ? hit.value : ""
+      });
+      if (!provisional.key) continue;
+    }
     const clash = existing.find((e) => sameWork(e, provisional)) ?? await getItem(provisional.key);
     if (clash) continue;
     const item = await putItem({
@@ -2166,6 +2322,41 @@ async function scanText(text) {
   }
   return created;
 }
+async function searchCandidates2(query, rows) {
+  const { searchCandidates: search } = await Promise.resolve().then(() => (init_search(), search_exports));
+  return search(query, { rows });
+}
+async function addCandidate(candidate) {
+  const rec = {
+    itemType: candidate.itemType ?? "journalArticle",
+    title: candidate.title ?? "",
+    authors: candidate.authors ?? [],
+    year: candidate.year ?? null,
+    container: candidate.container ?? "",
+    volume: candidate.volume ?? "",
+    issue: candidate.issue ?? "",
+    pages: candidate.pages ?? "",
+    doi: candidate.doi ?? ""
+  };
+  const provisional = buildItem(rec);
+  if (!provisional.key) throw failure("no_metadata", "\u5019\u9009\u7F3A\u5C11\u53EF\u8BC6\u522B\u4FE1\u606F");
+  const clash = await getItem(provisional.key);
+  if (clash) {
+    emitItem(clash);
+    return clash;
+  }
+  const item = await putItem({
+    ...provisional,
+    kind: candidate.doi ? "doi" : "title",
+    rawValue: candidate.doi || rec.title,
+    display: rec.title,
+    record: rec,
+    state: "resolved",
+    createdAt: Date.now()
+  });
+  emitItem(item);
+  return item;
+}
 async function resolveItem(key) {
   const item = await getItem(key);
   if (!item) throw failure("not_found", "\u6761\u76EE\u4E0D\u5B58\u5728");
@@ -2174,6 +2365,52 @@ async function resolveItem(key) {
   await update(key, { state: "resolving", error: null });
   const config = await loadConfig();
   try {
+    if (item.kind === "url") {
+      const { resolveUrlPage: resolveUrlPage2 } = await Promise.resolve().then(() => (init_url(), url_exports));
+      const page = await resolveUrlPage2(item.rawValue || item.title, { timeoutMs: 2e4 });
+      if (page.doi) {
+        const record2 = await resolveIdentifier({ kind: "doi", value: page.doi }, { timeoutMs: 2e4, unpaywallEmail: config.unpaywallEmail });
+        if (record2) {
+          const merged2 = buildItem({ ...item, ...record2 });
+          const updated2 = await patchItem(key, {
+            ...merged2,
+            key,
+            state: "resolved",
+            record: record2,
+            url: item.rawValue || page.url,
+            error: null,
+            updatedAt: Date.now()
+          });
+          emitItem(updated2);
+          await finishTask(task, "done", "\u5143\u6570\u636E\u89E3\u6790\u5B8C\u6210");
+          return updated2;
+        }
+      }
+      if (page.title) {
+        const record2 = {
+          itemType: "webpage",
+          title: page.title,
+          authors: page.authors ?? [],
+          year: page.year ?? null,
+          url: item.rawValue || page.url,
+          doi: page.doi
+        };
+        const updated2 = await patchItem(key, {
+          key,
+          state: "resolved",
+          record: record2,
+          title: page.title,
+          error: null,
+          updatedAt: Date.now()
+        });
+        emitItem(updated2);
+        await finishTask(task, "done", "\u5DF2\u4ECE\u9875\u9762\u89E3\u6790\u5143\u6570\u636E");
+        return updated2;
+      }
+      await update(key, { state: "resolve_failed", error: failure("no_metadata", page.error ? `\u65E0\u6CD5\u8BBF\u95EE\u9875\u9762\uFF1A${page.error}` : void 0) });
+      await finishTask(task, "failed", "\u65E0\u6CD5\u4ECE\u9875\u9762\u89E3\u6790\u5143\u6570\u636E");
+      return getItem(key);
+    }
     const record = await withRetry(
       () => resolveIdentifier(
         { kind: item.kind, value: item.rawValue || item.doi || item.arxiv || item.isbn || item.pmid || item.title },
@@ -2702,7 +2939,12 @@ async function handleEvents(req, res) {
   }
 }
 async function servePdf(req, res, key) {
-  const path = safePdfPath(key);
+  let decoded = key;
+  try {
+    decoded = decodeURIComponent(key);
+  } catch {
+  }
+  const path = safePdfPath(decoded);
   if (!path) {
     writeJson(res, 400, { error: "invalid key" });
     return;
@@ -2878,6 +3120,18 @@ async function handler(req, res) {
       writeJson(res, 200, result);
       return;
     }
+    if (head === "search" && methodOk(req, "POST")) {
+      const body = await readJsonBody2(req);
+      const candidates2 = await searchCandidates2(String(body?.q ?? ""), Number(body?.rows ?? 8));
+      writeJson(res, 200, { candidates: candidates2 });
+      return;
+    }
+    if (head === "add-candidate" && methodOk(req, "POST")) {
+      const body = await readJsonBody2(req);
+      const item = await addCandidate(body?.candidate ?? {});
+      writeJson(res, 200, { item });
+      return;
+    }
     if (head === "drop" && methodOk(req, "POST")) {
       const url = new URL(req.url ?? "/", "http://127.0.0.1");
       const filename = url.searchParams.get("filename") || "dropped.pdf";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@a9i5k4/dsh-literature",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "DSH Literature 文献侧窗：在 DeepSeek Harness 侧边栏识别 DOI/arXiv/标题、抓取元数据与全文、写入本地文献库或导出目录，并提供内置 PDF 阅读器（缩放/翻页/目录/搜索/高亮/笔记）。",
   "type": "module",
   "main": "lib/index.js",