@a9i5k4/dsh-literature 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +40 -13
- package/lib/index.js +545 -241
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -554,6 +554,12 @@ function extractIdentifiers(text, options = {}) {
|
|
|
554
554
|
while ((m = ISBN.exec(source)) !== null) {
|
|
555
555
|
push(out, seen, { kind: "isbn", value: m[1].replace(/[-\s]/g, ""), display: m[1], index: offset + m.index, confidence: 0.85 + confidenceBonus });
|
|
556
556
|
}
|
|
557
|
+
URL_RE.lastIndex = 0;
|
|
558
|
+
while ((m = URL_RE.exec(source)) !== null) {
|
|
559
|
+
const raw = trimRight(m[0]);
|
|
560
|
+
if (/doi\.org|arxiv\.org|dx\.doi/i.test(raw)) continue;
|
|
561
|
+
push(out, seen, { kind: "url", value: raw, display: raw, index: offset + m.index, confidence: 0.9 + confidenceBonus });
|
|
562
|
+
}
|
|
557
563
|
};
|
|
558
564
|
scan(src);
|
|
559
565
|
for (const url of linkTargets) scan(url, 0, 0.02);
|
|
@@ -571,7 +577,7 @@ function extractIdentifiers(text, options = {}) {
|
|
|
571
577
|
}
|
|
572
578
|
return out.sort((a, b) => a.index - b.index);
|
|
573
579
|
}
|
|
574
|
-
var TRIM_RIGHT, CJK_PUNCT, DOI_TAIL, DOI_CORE, DOI_HINT, DOI_LABEL, ARXIV_NEW, ARXIV_OLD, ARXIV_URL, PMID, ISBN, QUOTED, QUOTED_FALLBACK;
|
|
580
|
+
var TRIM_RIGHT, CJK_PUNCT, DOI_TAIL, DOI_CORE, DOI_HINT, DOI_LABEL, ARXIV_NEW, ARXIV_OLD, URL_RE, ARXIV_URL, PMID, ISBN, QUOTED, QUOTED_FALLBACK;
|
|
575
581
|
var init_identifiers = __esm({
|
|
576
582
|
"src/node/extract/identifiers.js"() {
|
|
577
583
|
TRIM_RIGHT = /[.,;:!?。、,;:!?…—>'">)》\]]+$/;
|
|
@@ -582,6 +588,7 @@ var init_identifiers = __esm({
|
|
|
582
588
|
DOI_LABEL = new RegExp(`\\bDOI\\s*[:\uFF1A]\\s*(10\\.\\d{4,9}\\/${DOI_TAIL})`, "gi");
|
|
583
589
|
ARXIV_NEW = /\barXiv\s*[:. ]?\s*(\d{4}\.\d{4,5})(v\d+)?\b/gi;
|
|
584
590
|
ARXIV_OLD = /\barXiv\s*[:. ]?\s*([a-z][a-z-]*(?:\.[A-Z]{2})?\/\d{7})(v\d+)?\b/gi;
|
|
591
|
+
URL_RE = /\bhttps?:\/\/[^\s<>"']+/gi;
|
|
585
592
|
ARXIV_URL = /arxiv\.org\/(?:abs|pdf)\/([^\s"'?#>]+?)(?:\.pdf)?(?=[\s"'?#>()]|$)/gi;
|
|
586
593
|
PMID = /\bPMID\s*[::]?\s*(\d{1,8})\b/gi;
|
|
587
594
|
ISBN = /\bISBN(?:-1[03])?\s*[::]?\s*((?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx])\b/gi;
|
|
@@ -1856,12 +1863,368 @@ var init_local_api = __esm({
|
|
|
1856
1863
|
}
|
|
1857
1864
|
});
|
|
1858
1865
|
|
|
1866
|
+
// src/node/metadata/search.js
|
|
1867
|
+
var search_exports = {};
|
|
1868
|
+
__export(search_exports, {
|
|
1869
|
+
searchCandidates: () => searchCandidates
|
|
1870
|
+
});
|
|
1871
|
+
function crossrefType(t) {
|
|
1872
|
+
switch (t) {
|
|
1873
|
+
case "journal-article":
|
|
1874
|
+
return "journalArticle";
|
|
1875
|
+
case "book":
|
|
1876
|
+
return "book";
|
|
1877
|
+
case "book-chapter":
|
|
1878
|
+
return "bookSection";
|
|
1879
|
+
case "proceedings-article":
|
|
1880
|
+
return "conferencePaper";
|
|
1881
|
+
case "posted-content":
|
|
1882
|
+
return "preprint";
|
|
1883
|
+
case "dissertation":
|
|
1884
|
+
return "thesis";
|
|
1885
|
+
case "report":
|
|
1886
|
+
return "report";
|
|
1887
|
+
case "webpage":
|
|
1888
|
+
return "webpage";
|
|
1889
|
+
default:
|
|
1890
|
+
return "journalArticle";
|
|
1891
|
+
}
|
|
1892
|
+
}
|
|
1893
|
+
async function searchCandidates(query, { rows = 8 } = {}) {
|
|
1894
|
+
const q = String(query ?? "").trim();
|
|
1895
|
+
if (!q) return [];
|
|
1896
|
+
const url = `https://api.crossref.org/works?query.bibliographic=${encodeURIComponent(q)}&rows=${Math.max(1, Math.min(rows, 20))}&select=DOI,title,author,issued,container-title,type,volume,issue,page`;
|
|
1897
|
+
let data;
|
|
1898
|
+
try {
|
|
1899
|
+
data = await httpGetJson(url);
|
|
1900
|
+
} catch {
|
|
1901
|
+
return [];
|
|
1902
|
+
}
|
|
1903
|
+
const out = [];
|
|
1904
|
+
for (const it of data?.message?.items ?? []) {
|
|
1905
|
+
const title = it.title?.[0];
|
|
1906
|
+
if (!title) continue;
|
|
1907
|
+
out.push({
|
|
1908
|
+
doi: it.DOI ?? "",
|
|
1909
|
+
title,
|
|
1910
|
+
authors: (it.author ?? []).map((a) => ({ firstName: a.given ?? "", lastName: a.family ?? "" })),
|
|
1911
|
+
year: it.issued?.["date-parts"]?.[0]?.[0] ?? null,
|
|
1912
|
+
container: it["container-title"]?.[0] ?? "",
|
|
1913
|
+
volume: it.volume ?? "",
|
|
1914
|
+
issue: it.issue ?? "",
|
|
1915
|
+
pages: it.page ?? "",
|
|
1916
|
+
itemType: crossrefType(it.type)
|
|
1917
|
+
});
|
|
1918
|
+
}
|
|
1919
|
+
return out;
|
|
1920
|
+
}
|
|
1921
|
+
var init_search = __esm({
|
|
1922
|
+
"src/node/metadata/search.js"() {
|
|
1923
|
+
init_net();
|
|
1924
|
+
}
|
|
1925
|
+
});
|
|
1926
|
+
|
|
1927
|
+
// src/node/metadata/url.js
|
|
1928
|
+
var url_exports = {};
|
|
1929
|
+
__export(url_exports, {
|
|
1930
|
+
resolveUrlPage: () => resolveUrlPage
|
|
1931
|
+
});
|
|
1932
|
+
function firstMatch(html, patterns) {
|
|
1933
|
+
for (const re of patterns) {
|
|
1934
|
+
const m = re.exec(html);
|
|
1935
|
+
if (m?.[1]) return m[1].trim();
|
|
1936
|
+
}
|
|
1937
|
+
return "";
|
|
1938
|
+
}
|
|
1939
|
+
async function resolveUrlPage(url, { timeoutMs = 15e3 } = {}) {
|
|
1940
|
+
let html = "";
|
|
1941
|
+
let error2 = "";
|
|
1942
|
+
try {
|
|
1943
|
+
const res = await httpGet(url, {
|
|
1944
|
+
timeoutMs,
|
|
1945
|
+
accept: "text/html,application/xhtml+xml,*/*;q=0.8",
|
|
1946
|
+
// Many publisher pages 403 bare requests; a browser UA keeps them open.
|
|
1947
|
+
headers: { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" }
|
|
1948
|
+
});
|
|
1949
|
+
html = await res.text();
|
|
1950
|
+
} catch (e) {
|
|
1951
|
+
error2 = e?.message ?? String(e);
|
|
1952
|
+
}
|
|
1953
|
+
const title = firstMatch(html, META_PATTERNS);
|
|
1954
|
+
const doi = firstMatch(html, DOI_PATTERNS);
|
|
1955
|
+
const author = firstMatch(html, AUTHOR_PATTERNS);
|
|
1956
|
+
const year = firstMatch(html, DATE_PATTERNS);
|
|
1957
|
+
const cleanedTitle = title.replace(/[|–-].*$/, "").trim();
|
|
1958
|
+
return {
|
|
1959
|
+
title: cleanedTitle || "",
|
|
1960
|
+
doi,
|
|
1961
|
+
authors: author ? [{ firstName: "", lastName: author }] : [],
|
|
1962
|
+
year: year ? Number(year) : null,
|
|
1963
|
+
url,
|
|
1964
|
+
error: error2
|
|
1965
|
+
};
|
|
1966
|
+
}
|
|
1967
|
+
var META_PATTERNS, DOI_PATTERNS, AUTHOR_PATTERNS, DATE_PATTERNS;
|
|
1968
|
+
var init_url = __esm({
|
|
1969
|
+
"src/node/metadata/url.js"() {
|
|
1970
|
+
init_net();
|
|
1971
|
+
META_PATTERNS = [
|
|
1972
|
+
// <meta name="citation_title" content="..."> — order of attrs varies
|
|
1973
|
+
/<meta[^>]+name=["']citation_title["'][^>]+content=["']([^"']+)["']/i,
|
|
1974
|
+
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_title["']/i,
|
|
1975
|
+
// Open Graph
|
|
1976
|
+
/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i,
|
|
1977
|
+
/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["']/i,
|
|
1978
|
+
// fallback
|
|
1979
|
+
/<title[^>]*>([^<]+)<\/title>/i
|
|
1980
|
+
];
|
|
1981
|
+
DOI_PATTERNS = [
|
|
1982
|
+
/<meta[^>]+name=["']citation_doi["'][^>]+content=["']([^"']+)["']/i,
|
|
1983
|
+
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_doi["']/i
|
|
1984
|
+
];
|
|
1985
|
+
AUTHOR_PATTERNS = [
|
|
1986
|
+
/<meta[^>]+name=["']citation_author["'][^>]+content=["']([^"']+)["']/i,
|
|
1987
|
+
/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_author["']/i
|
|
1988
|
+
];
|
|
1989
|
+
DATE_PATTERNS = [
|
|
1990
|
+
/<meta[^>]+name=["']citation_publication_date["'][^>]+content=["'](\d{4})/i,
|
|
1991
|
+
/<meta[^>]+content=["'](\d{4})["'][^>]+name=["']citation_publication_date["']/i
|
|
1992
|
+
];
|
|
1993
|
+
}
|
|
1994
|
+
});
|
|
1995
|
+
|
|
1996
|
+
// src/node/importer.js
|
|
1997
|
+
var importer_exports = {};
|
|
1998
|
+
__export(importer_exports, {
|
|
1999
|
+
importDir: () => importDir,
|
|
2000
|
+
importFromZotero: () => importFromZotero,
|
|
2001
|
+
startWatcher: () => startWatcher,
|
|
2002
|
+
stopWatcher: () => stopWatcher,
|
|
2003
|
+
titleFromFilename: () => titleFromFilename,
|
|
2004
|
+
zoteroItemToRecord: () => zoteroItemToRecord
|
|
2005
|
+
});
|
|
2006
|
+
import { readdir as readdir2, readFile as readFile3, stat as stat2 } from "node:fs/promises";
|
|
2007
|
+
import { join as join4, resolve as resolve4, extname, basename } from "node:path";
|
|
2008
|
+
import { writeFile as writeFile3, mkdir as mkdir3 } from "node:fs/promises";
|
|
2009
|
+
import { dirname as dirname2 } from "node:path";
|
|
2010
|
+
function titleFromFilename(name2) {
|
|
2011
|
+
let t = String(name2).replace(/\.pdf$/i, "");
|
|
2012
|
+
const doi = /(10\.\d{4,9}[\/_][^\s]+)/i.exec(t);
|
|
2013
|
+
if (doi) return { kind: "doi", value: doi[1].replace(/_/g, "/") };
|
|
2014
|
+
const arxiv = /\b(\d{4}\.\d{4,5})(v\d+)?\b/.exec(t);
|
|
2015
|
+
if (arxiv) return { kind: "arxiv", value: arxiv[1] + (arxiv[2] ?? "") };
|
|
2016
|
+
t = t.replace(/^\[\d+\]\s*/, "").replace(/\s*\(?\d{4}[a-z]?\)?\s*$/i, "").replace(/[_\-]+/g, " ").replace(/^\d+[\s.]+/, "").trim();
|
|
2017
|
+
if (!t) return null;
|
|
2018
|
+
return { kind: "title", value: t };
|
|
2019
|
+
}
|
|
2020
|
+
function pathForPdfKey(key) {
|
|
2021
|
+
return join4(PDF_DIR, `${key.replace(/[^\w.-]+/g, "_")}.pdf`);
|
|
2022
|
+
}
|
|
2023
|
+
async function savePdfBuffer(key, buffer) {
|
|
2024
|
+
const path = pathForPdfKey(key);
|
|
2025
|
+
await mkdir3(dirname2(path), { recursive: true });
|
|
2026
|
+
await writeFile3(path, buffer);
|
|
2027
|
+
return { path, size: buffer.length, source: "library-import" };
|
|
2028
|
+
}
|
|
2029
|
+
async function importDir(dir, { autoResolve = true } = {}) {
|
|
2030
|
+
const target = resolve4(dir || "");
|
|
2031
|
+
if (!target) throw Object.assign(new Error("\u672A\u914D\u7F6E\u5BFC\u5165\u6587\u4EF6\u5939"), { code: "no_dir" });
|
|
2032
|
+
let files;
|
|
2033
|
+
try {
|
|
2034
|
+
files = await readdir2(target);
|
|
2035
|
+
} catch (e) {
|
|
2036
|
+
throw Object.assign(new Error(`\u65E0\u6CD5\u8BFB\u53D6\u6587\u4EF6\u5939 ${target}`), { code: "no_dir", cause: e });
|
|
2037
|
+
}
|
|
2038
|
+
const imported = [];
|
|
2039
|
+
const skipped = [];
|
|
2040
|
+
for (const name2 of files.filter((f) => extname(f).toLowerCase() === ".pdf")) {
|
|
2041
|
+
const abs = join4(target, name2);
|
|
2042
|
+
const seen = await getImportedFile(abs);
|
|
2043
|
+
let st;
|
|
2044
|
+
try {
|
|
2045
|
+
st = await stat2(abs);
|
|
2046
|
+
} catch {
|
|
2047
|
+
continue;
|
|
2048
|
+
}
|
|
2049
|
+
if (seen && seen.mtimeMs === st.mtimeMs) {
|
|
2050
|
+
skipped.push({ file: name2, reason: "already-imported" });
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
let item;
|
|
2054
|
+
try {
|
|
2055
|
+
const hit = titleFromFilename(name2);
|
|
2056
|
+
if (!hit) {
|
|
2057
|
+
skipped.push({ file: name2, reason: "unparseable-name" });
|
|
2058
|
+
continue;
|
|
2059
|
+
}
|
|
2060
|
+
const provisional = buildItem({
|
|
2061
|
+
doi: hit.kind === "doi" ? hit.value : "",
|
|
2062
|
+
arxiv: hit.kind === "arxiv" ? hit.value : "",
|
|
2063
|
+
title: hit.kind === "title" ? hit.value : ""
|
|
2064
|
+
});
|
|
2065
|
+
if (!provisional.key) {
|
|
2066
|
+
skipped.push({ file: name2, reason: "no-identity" });
|
|
2067
|
+
continue;
|
|
2068
|
+
}
|
|
2069
|
+
const clash = await getItem(provisional.key);
|
|
2070
|
+
if (clash) {
|
|
2071
|
+
await addImportedFile(abs, st.mtimeMs);
|
|
2072
|
+
skipped.push({ file: name2, reason: "duplicate" });
|
|
2073
|
+
continue;
|
|
2074
|
+
}
|
|
2075
|
+
item = await putItem({
|
|
2076
|
+
...provisional,
|
|
2077
|
+
kind: hit.kind,
|
|
2078
|
+
rawValue: hit.value,
|
|
2079
|
+
display: hit.value,
|
|
2080
|
+
title: provisional.title || hit.value,
|
|
2081
|
+
state: "discovered",
|
|
2082
|
+
sourceFile: abs,
|
|
2083
|
+
createdAt: Date.now()
|
|
2084
|
+
});
|
|
2085
|
+
emitItem(item);
|
|
2086
|
+
} catch (e) {
|
|
2087
|
+
warn(`import ${name2} failed:`, e.message);
|
|
2088
|
+
skipped.push({ file: name2, reason: e.message });
|
|
2089
|
+
continue;
|
|
2090
|
+
}
|
|
2091
|
+
if (autoResolve) {
|
|
2092
|
+
try {
|
|
2093
|
+
const { resolveItem: resolveItem2 } = await Promise.resolve().then(() => (init_pipeline(), pipeline_exports));
|
|
2094
|
+
item = await resolveItem2(item.key);
|
|
2095
|
+
} catch (e) {
|
|
2096
|
+
warn(`auto-resolve ${name2} failed:`, e.message);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
await addImportedFile(abs, st.mtimeMs);
|
|
2100
|
+
imported.push(item?.key ?? provisionalKeyOf(item));
|
|
2101
|
+
}
|
|
2102
|
+
return { imported, skipped, dir: target };
|
|
2103
|
+
}
|
|
2104
|
+
function provisionalKeyOf(item) {
|
|
2105
|
+
return item?.key ?? "";
|
|
2106
|
+
}
|
|
2107
|
+
function zoteroItemToRecord(it) {
|
|
2108
|
+
const d = it?.data ?? it ?? {};
|
|
2109
|
+
const year = /^(\d{4})/.exec(String(d.date ?? ""))?.[1];
|
|
2110
|
+
return {
|
|
2111
|
+
source: "zotero",
|
|
2112
|
+
itemType: d.itemType ?? "journalArticle",
|
|
2113
|
+
title: d.title ?? "",
|
|
2114
|
+
authors: (d.creators ?? []).map((c) => ({
|
|
2115
|
+
creatorType: c.creatorType ?? "author",
|
|
2116
|
+
firstName: c.firstName ?? "",
|
|
2117
|
+
lastName: c.lastName ?? ""
|
|
2118
|
+
})),
|
|
2119
|
+
year: year ? Number(year) : null,
|
|
2120
|
+
container: d.publicationTitle ?? d.bookTitle ?? d.proceedingsTitle ?? "",
|
|
2121
|
+
publisher: d.publisher ?? "",
|
|
2122
|
+
volume: d.volume ?? "",
|
|
2123
|
+
issue: d.issue ?? "",
|
|
2124
|
+
pages: d.pages ?? "",
|
|
2125
|
+
doi: d.DOI ?? "",
|
|
2126
|
+
isbn: d.ISBN ?? "",
|
|
2127
|
+
url: d.url ?? "",
|
|
2128
|
+
abstract: d.abstractNote ?? ""
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
async function importFromZotero({ limit = 50 } = {}) {
|
|
2132
|
+
const items = await searchItems("", { limit });
|
|
2133
|
+
const imported = [];
|
|
2134
|
+
const skipped = [];
|
|
2135
|
+
for (const it of items) {
|
|
2136
|
+
const rec = zoteroItemToRecord(it);
|
|
2137
|
+
if (!rec.title) {
|
|
2138
|
+
skipped.push({ reason: "no-title" });
|
|
2139
|
+
continue;
|
|
2140
|
+
}
|
|
2141
|
+
const provisional = buildItem(rec);
|
|
2142
|
+
if (!provisional.key) {
|
|
2143
|
+
skipped.push({ reason: "no-identity" });
|
|
2144
|
+
continue;
|
|
2145
|
+
}
|
|
2146
|
+
const existing = await getItem(provisional.key);
|
|
2147
|
+
if (existing) {
|
|
2148
|
+
skipped.push({ reason: "duplicate" });
|
|
2149
|
+
continue;
|
|
2150
|
+
}
|
|
2151
|
+
let pdf = null;
|
|
2152
|
+
try {
|
|
2153
|
+
const children = await getItemChildren(it.key);
|
|
2154
|
+
const attach = (children ?? []).find(
|
|
2155
|
+
(c) => c.itemType === "attachment" && /pdf/i.test(c.contentType ?? c.contentType ?? "")
|
|
2156
|
+
);
|
|
2157
|
+
if (attach?.key) {
|
|
2158
|
+
const { buffer } = await getFileBuffer(attach.key).catch(() => ({ buffer: null }));
|
|
2159
|
+
if (buffer?.length && buffer.subarray(0, 5).equals(PDF_MAGIC2)) {
|
|
2160
|
+
pdf = await savePdfBuffer(provisional.key, buffer);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
} catch (e) {
|
|
2164
|
+
warn(`attachment import failed for ${it.key}:`, e.message);
|
|
2165
|
+
}
|
|
2166
|
+
const item = await putItem({
|
|
2167
|
+
...provisional,
|
|
2168
|
+
record: rec,
|
|
2169
|
+
state: pdf ? "fetched" : "resolved",
|
|
2170
|
+
pdf,
|
|
2171
|
+
createdAt: Date.now()
|
|
2172
|
+
});
|
|
2173
|
+
emitItem(item);
|
|
2174
|
+
imported.push(item.key);
|
|
2175
|
+
}
|
|
2176
|
+
return { imported, skipped, count: imported.length };
|
|
2177
|
+
}
|
|
2178
|
+
function startWatcher() {
|
|
2179
|
+
if (watcherTimer) return () => stopWatcher();
|
|
2180
|
+
let stopping = false;
|
|
2181
|
+
const sweep = async () => {
|
|
2182
|
+
if (stopping) return;
|
|
2183
|
+
const config = await loadConfig();
|
|
2184
|
+
if (!config.watchImport || !config.importDir) return;
|
|
2185
|
+
try {
|
|
2186
|
+
const r = await importDir(config.importDir, { autoResolve: config.autoResolve !== false });
|
|
2187
|
+
if (r.imported.length) log(`folder watch imported ${r.imported.length} new file(s)`);
|
|
2188
|
+
} catch (e) {
|
|
2189
|
+
warn("folder watch sweep failed:", e.message);
|
|
2190
|
+
}
|
|
2191
|
+
};
|
|
2192
|
+
const initial = setTimeout(sweep, 5e3);
|
|
2193
|
+
initial.unref?.();
|
|
2194
|
+
const timer = setInterval(sweep, 3e4);
|
|
2195
|
+
timer.unref?.();
|
|
2196
|
+
watcherTimer = { timer, initial };
|
|
2197
|
+
return () => stopWatcher();
|
|
2198
|
+
}
|
|
2199
|
+
function stopWatcher() {
|
|
2200
|
+
if (!watcherTimer) return;
|
|
2201
|
+
clearInterval(watcherTimer.timer);
|
|
2202
|
+
clearTimeout(watcherTimer.initial);
|
|
2203
|
+
watcherTimer = null;
|
|
2204
|
+
}
|
|
2205
|
+
var PDF_MAGIC2, watcherTimer;
|
|
2206
|
+
var init_importer = __esm({
|
|
2207
|
+
"src/node/importer.js"() {
|
|
2208
|
+
init_config();
|
|
2209
|
+
init_db();
|
|
2210
|
+
init_sse();
|
|
2211
|
+
init_dedupe();
|
|
2212
|
+
init_metadata();
|
|
2213
|
+
init_local_api();
|
|
2214
|
+
init_log();
|
|
2215
|
+
PDF_MAGIC2 = Buffer.from("%PDF-", "latin1");
|
|
2216
|
+
watcherTimer = null;
|
|
2217
|
+
}
|
|
2218
|
+
});
|
|
2219
|
+
|
|
1859
2220
|
// src/node/pipeline.js
|
|
1860
2221
|
var pipeline_exports = {};
|
|
1861
2222
|
__export(pipeline_exports, {
|
|
2223
|
+
addCandidate: () => addCandidate,
|
|
1862
2224
|
arxivBase: () => arxivBase,
|
|
1863
2225
|
discardItem: () => discardItem,
|
|
1864
2226
|
fetchItemPdf: () => fetchItemPdf,
|
|
2227
|
+
importDroppedPdf: () => importDroppedPdf,
|
|
1865
2228
|
importPdf: () => importPdf,
|
|
1866
2229
|
log: () => log,
|
|
1867
2230
|
normalizeDoi: () => normalizeDoi,
|
|
@@ -1870,11 +2233,12 @@ __export(pipeline_exports, {
|
|
|
1870
2233
|
retryItem: () => retryItem,
|
|
1871
2234
|
saveItem: () => saveItem,
|
|
1872
2235
|
scanText: () => scanText,
|
|
2236
|
+
searchCandidates: () => searchCandidates2,
|
|
1873
2237
|
startTask: () => startTask
|
|
1874
2238
|
});
|
|
1875
2239
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1876
|
-
import { writeFile as
|
|
1877
|
-
import { join as
|
|
2240
|
+
import { writeFile as writeFile4, unlink, mkdir as mkdir4 } from "node:fs/promises";
|
|
2241
|
+
import { join as join5, dirname as dirname3 } from "node:path";
|
|
1878
2242
|
function failure(code, message, extra = {}) {
|
|
1879
2243
|
return {
|
|
1880
2244
|
code,
|
|
@@ -1885,7 +2249,7 @@ function failure(code, message, extra = {}) {
|
|
|
1885
2249
|
};
|
|
1886
2250
|
}
|
|
1887
2251
|
function pdfPathFor(key) {
|
|
1888
|
-
return
|
|
2252
|
+
return join5(PDF_DIR, `${key.replace(/[^\w.-]+/g, "_")}.pdf`);
|
|
1889
2253
|
}
|
|
1890
2254
|
async function update(key, patch) {
|
|
1891
2255
|
const next = await patchItem(key, patch);
|
|
@@ -1916,14 +2280,21 @@ async function scanText(text) {
|
|
|
1916
2280
|
const existing = await listItems();
|
|
1917
2281
|
const created = [];
|
|
1918
2282
|
for (const hit of found) {
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
2283
|
+
let provisional;
|
|
2284
|
+
if (hit.kind === "url") {
|
|
2285
|
+
provisional = buildItem({ title: hit.value });
|
|
2286
|
+
if (!provisional.key) continue;
|
|
2287
|
+
provisional = { ...provisional, title: hit.value };
|
|
2288
|
+
} else {
|
|
2289
|
+
provisional = buildItem({
|
|
2290
|
+
doi: hit.kind === "doi" ? hit.value : "",
|
|
2291
|
+
arxiv: hit.kind === "arxiv" ? hit.value : "",
|
|
2292
|
+
isbn: hit.kind === "isbn" ? hit.value : "",
|
|
2293
|
+
pmid: hit.kind === "pmid" ? hit.value : "",
|
|
2294
|
+
title: hit.kind === "title" ? hit.value : ""
|
|
2295
|
+
});
|
|
2296
|
+
if (!provisional.key) continue;
|
|
2297
|
+
}
|
|
1927
2298
|
const clash = existing.find((e) => sameWork(e, provisional)) ?? await getItem(provisional.key);
|
|
1928
2299
|
if (clash) continue;
|
|
1929
2300
|
const item = await putItem({
|
|
@@ -1941,6 +2312,41 @@ async function scanText(text) {
|
|
|
1941
2312
|
}
|
|
1942
2313
|
return created;
|
|
1943
2314
|
}
|
|
2315
|
+
async function searchCandidates2(query, rows) {
|
|
2316
|
+
const { searchCandidates: search } = await Promise.resolve().then(() => (init_search(), search_exports));
|
|
2317
|
+
return search(query, { rows });
|
|
2318
|
+
}
|
|
2319
|
+
async function addCandidate(candidate) {
|
|
2320
|
+
const rec = {
|
|
2321
|
+
itemType: candidate.itemType ?? "journalArticle",
|
|
2322
|
+
title: candidate.title ?? "",
|
|
2323
|
+
authors: candidate.authors ?? [],
|
|
2324
|
+
year: candidate.year ?? null,
|
|
2325
|
+
container: candidate.container ?? "",
|
|
2326
|
+
volume: candidate.volume ?? "",
|
|
2327
|
+
issue: candidate.issue ?? "",
|
|
2328
|
+
pages: candidate.pages ?? "",
|
|
2329
|
+
doi: candidate.doi ?? ""
|
|
2330
|
+
};
|
|
2331
|
+
const provisional = buildItem(rec);
|
|
2332
|
+
if (!provisional.key) throw failure("no_metadata", "\u5019\u9009\u7F3A\u5C11\u53EF\u8BC6\u522B\u4FE1\u606F");
|
|
2333
|
+
const clash = await getItem(provisional.key);
|
|
2334
|
+
if (clash) {
|
|
2335
|
+
emitItem(clash);
|
|
2336
|
+
return clash;
|
|
2337
|
+
}
|
|
2338
|
+
const item = await putItem({
|
|
2339
|
+
...provisional,
|
|
2340
|
+
kind: candidate.doi ? "doi" : "title",
|
|
2341
|
+
rawValue: candidate.doi || rec.title,
|
|
2342
|
+
display: rec.title,
|
|
2343
|
+
record: rec,
|
|
2344
|
+
state: "resolved",
|
|
2345
|
+
createdAt: Date.now()
|
|
2346
|
+
});
|
|
2347
|
+
emitItem(item);
|
|
2348
|
+
return item;
|
|
2349
|
+
}
|
|
1944
2350
|
async function resolveItem(key) {
|
|
1945
2351
|
const item = await getItem(key);
|
|
1946
2352
|
if (!item) throw failure("not_found", "\u6761\u76EE\u4E0D\u5B58\u5728");
|
|
@@ -1949,6 +2355,52 @@ async function resolveItem(key) {
|
|
|
1949
2355
|
await update(key, { state: "resolving", error: null });
|
|
1950
2356
|
const config = await loadConfig();
|
|
1951
2357
|
try {
|
|
2358
|
+
if (item.kind === "url") {
|
|
2359
|
+
const { resolveUrlPage: resolveUrlPage2 } = await Promise.resolve().then(() => (init_url(), url_exports));
|
|
2360
|
+
const page = await resolveUrlPage2(item.rawValue || item.title, { timeoutMs: 2e4 });
|
|
2361
|
+
if (page.doi) {
|
|
2362
|
+
const record2 = await resolveIdentifier({ kind: "doi", value: page.doi }, { timeoutMs: 2e4, unpaywallEmail: config.unpaywallEmail });
|
|
2363
|
+
if (record2) {
|
|
2364
|
+
const merged2 = buildItem({ ...item, ...record2 });
|
|
2365
|
+
const updated2 = await patchItem(key, {
|
|
2366
|
+
...merged2,
|
|
2367
|
+
key,
|
|
2368
|
+
state: "resolved",
|
|
2369
|
+
record: record2,
|
|
2370
|
+
url: item.rawValue || page.url,
|
|
2371
|
+
error: null,
|
|
2372
|
+
updatedAt: Date.now()
|
|
2373
|
+
});
|
|
2374
|
+
emitItem(updated2);
|
|
2375
|
+
await finishTask(task, "done", "\u5143\u6570\u636E\u89E3\u6790\u5B8C\u6210");
|
|
2376
|
+
return updated2;
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
if (page.title) {
|
|
2380
|
+
const record2 = {
|
|
2381
|
+
itemType: "webpage",
|
|
2382
|
+
title: page.title,
|
|
2383
|
+
authors: page.authors ?? [],
|
|
2384
|
+
year: page.year ?? null,
|
|
2385
|
+
url: item.rawValue || page.url,
|
|
2386
|
+
doi: page.doi
|
|
2387
|
+
};
|
|
2388
|
+
const updated2 = await patchItem(key, {
|
|
2389
|
+
key,
|
|
2390
|
+
state: "resolved",
|
|
2391
|
+
record: record2,
|
|
2392
|
+
title: page.title,
|
|
2393
|
+
error: null,
|
|
2394
|
+
updatedAt: Date.now()
|
|
2395
|
+
});
|
|
2396
|
+
emitItem(updated2);
|
|
2397
|
+
await finishTask(task, "done", "\u5DF2\u4ECE\u9875\u9762\u89E3\u6790\u5143\u6570\u636E");
|
|
2398
|
+
return updated2;
|
|
2399
|
+
}
|
|
2400
|
+
await update(key, { state: "resolve_failed", error: failure("no_metadata", page.error ? `\u65E0\u6CD5\u8BBF\u95EE\u9875\u9762\uFF1A${page.error}` : void 0) });
|
|
2401
|
+
await finishTask(task, "failed", "\u65E0\u6CD5\u4ECE\u9875\u9762\u89E3\u6790\u5143\u6570\u636E");
|
|
2402
|
+
return getItem(key);
|
|
2403
|
+
}
|
|
1952
2404
|
const record = await withRetry(
|
|
1953
2405
|
() => resolveIdentifier(
|
|
1954
2406
|
{ kind: item.kind, value: item.rawValue || item.doi || item.arxiv || item.isbn || item.pmid || item.title },
|
|
@@ -2001,8 +2453,8 @@ async function fetchItemPdf(key) {
|
|
|
2001
2453
|
{ ...config.retry, label: `fetch ${key}` }
|
|
2002
2454
|
);
|
|
2003
2455
|
const path = pdfPathFor(key);
|
|
2004
|
-
await
|
|
2005
|
-
await
|
|
2456
|
+
await mkdir4(dirname3(path), { recursive: true });
|
|
2457
|
+
await writeFile4(path, result.buffer);
|
|
2006
2458
|
const updated = await patchItem(key, {
|
|
2007
2459
|
state: "fetched",
|
|
2008
2460
|
pdf: { path, size: result.buffer.length, source: result.source, url: result.url },
|
|
@@ -2154,8 +2606,8 @@ async function importPdf(key, buffer, { filename = "imported.pdf", autoSave = tr
|
|
|
2154
2606
|
throw failure("network", "\u6240\u9009\u6587\u4EF6\u4E0D\u662F\u6709\u6548\u7684 PDF");
|
|
2155
2607
|
}
|
|
2156
2608
|
const path = pdfPathFor(key);
|
|
2157
|
-
await
|
|
2158
|
-
await
|
|
2609
|
+
await mkdir4(dirname3(path), { recursive: true });
|
|
2610
|
+
await writeFile4(path, buffer);
|
|
2159
2611
|
const updated = await update(key, {
|
|
2160
2612
|
state: "fetched",
|
|
2161
2613
|
pdf: { path, size: buffer.length, source: "local-import", url: "", filename },
|
|
@@ -2163,8 +2615,64 @@ async function importPdf(key, buffer, { filename = "imported.pdf", autoSave = tr
|
|
|
2163
2615
|
});
|
|
2164
2616
|
let saved = updated;
|
|
2165
2617
|
if (autoSave) {
|
|
2166
|
-
saved = await saveItem(key);
|
|
2618
|
+
saved = await saveItem(key, { mode: "builtin" });
|
|
2619
|
+
}
|
|
2620
|
+
return saved;
|
|
2621
|
+
}
|
|
2622
|
+
async function importDroppedPdf(buffer, { filename = "dropped.pdf" } = {}) {
|
|
2623
|
+
if (!buffer || buffer.length < 8 || !buffer.subarray(0, 5).equals(Buffer.from("%PDF-", "latin1"))) {
|
|
2624
|
+
throw failure("network", "\u62D6\u5165\u7684\u6587\u4EF6\u4E0D\u662F\u6709\u6548\u7684 PDF");
|
|
2625
|
+
}
|
|
2626
|
+
const { titleFromFilename: titleFromFilename2 } = await Promise.resolve().then(() => (init_importer(), importer_exports));
|
|
2627
|
+
const hit = titleFromFilename2(filename);
|
|
2628
|
+
const provisional = buildItem(
|
|
2629
|
+
hit ? {
|
|
2630
|
+
doi: hit.kind === "doi" ? hit.value : "",
|
|
2631
|
+
arxiv: hit.kind === "arxiv" ? hit.value : "",
|
|
2632
|
+
title: hit.kind === "title" ? hit.value : ""
|
|
2633
|
+
} : { title: String(filename).replace(/\.pdf$/i, "") }
|
|
2634
|
+
);
|
|
2635
|
+
if (!provisional.key) throw failure("network", "\u65E0\u6CD5\u4ECE\u6587\u4EF6\u540D\u8BC6\u522B\u8BE5\u6587\u732E");
|
|
2636
|
+
const clash = await getItem(provisional.key);
|
|
2637
|
+
if (clash) {
|
|
2638
|
+
const path2 = pdfPathFor(provisional.key);
|
|
2639
|
+
await mkdir4(dirname3(path2), { recursive: true });
|
|
2640
|
+
await writeFile4(path2, buffer);
|
|
2641
|
+
const updated = await patchItem(provisional.key, {
|
|
2642
|
+
state: "fetched",
|
|
2643
|
+
pdf: { path: path2, size: buffer.length, source: "local-import", url: "", filename },
|
|
2644
|
+
error: null,
|
|
2645
|
+
updatedAt: Date.now()
|
|
2646
|
+
});
|
|
2647
|
+
emitItem(updated);
|
|
2648
|
+
return updated;
|
|
2167
2649
|
}
|
|
2650
|
+
const path = pdfPathFor(provisional.key);
|
|
2651
|
+
await mkdir4(dirname3(path), { recursive: true });
|
|
2652
|
+
await writeFile4(path, buffer);
|
|
2653
|
+
let item = await putItem({
|
|
2654
|
+
...provisional,
|
|
2655
|
+
kind: hit?.kind ?? "title",
|
|
2656
|
+
rawValue: hit?.value ?? "",
|
|
2657
|
+
display: hit?.value ?? provisional.title,
|
|
2658
|
+
sourceFile: filename,
|
|
2659
|
+
state: "fetched",
|
|
2660
|
+
pdf: { path, size: buffer.length, source: "drop-import", url: "", filename },
|
|
2661
|
+
createdAt: Date.now()
|
|
2662
|
+
});
|
|
2663
|
+
emitItem(item);
|
|
2664
|
+
try {
|
|
2665
|
+
item = await resolveItem(item.key);
|
|
2666
|
+
} catch {
|
|
2667
|
+
}
|
|
2668
|
+
if (!item.record) {
|
|
2669
|
+
item = await patchItem(item.key, {
|
|
2670
|
+
record: { itemType: "journalArticle", title: item.title || provisional.title, authors: [], year: null },
|
|
2671
|
+
state: "resolved"
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
const saved = await saveItem(item.key, { mode: "builtin" });
|
|
2675
|
+
emitItem(saved);
|
|
2168
2676
|
return saved;
|
|
2169
2677
|
}
|
|
2170
2678
|
var FAILURE_MESSAGES;
|
|
@@ -2367,230 +2875,6 @@ var init_cite = __esm({
|
|
|
2367
2875
|
}
|
|
2368
2876
|
});
|
|
2369
2877
|
|
|
2370
|
-
// src/node/importer.js
|
|
2371
|
-
var importer_exports = {};
|
|
2372
|
-
__export(importer_exports, {
|
|
2373
|
-
importDir: () => importDir,
|
|
2374
|
-
importFromZotero: () => importFromZotero,
|
|
2375
|
-
startWatcher: () => startWatcher,
|
|
2376
|
-
stopWatcher: () => stopWatcher,
|
|
2377
|
-
titleFromFilename: () => titleFromFilename,
|
|
2378
|
-
zoteroItemToRecord: () => zoteroItemToRecord
|
|
2379
|
-
});
|
|
2380
|
-
import { readdir as readdir2, readFile as readFile3, stat as stat2 } from "node:fs/promises";
|
|
2381
|
-
import { join as join5, resolve as resolve4, extname, basename } from "node:path";
|
|
2382
|
-
import { writeFile as writeFile4, mkdir as mkdir4 } from "node:fs/promises";
|
|
2383
|
-
import { dirname as dirname3 } from "node:path";
|
|
2384
|
-
function titleFromFilename(name2) {
|
|
2385
|
-
let t = String(name2).replace(/\.pdf$/i, "");
|
|
2386
|
-
const doi = /(10\.\d{4,9}[\/_][^\s]+)/i.exec(t);
|
|
2387
|
-
if (doi) return { kind: "doi", value: doi[1].replace(/_/g, "/") };
|
|
2388
|
-
const arxiv = /\b(\d{4}\.\d{4,5})(v\d+)?\b/.exec(t);
|
|
2389
|
-
if (arxiv) return { kind: "arxiv", value: arxiv[1] + (arxiv[2] ?? "") };
|
|
2390
|
-
t = t.replace(/^\[\d+\]\s*/, "").replace(/\s*\(?\d{4}[a-z]?\)?\s*$/i, "").replace(/[_\-]+/g, " ").replace(/^\d+[\s.]+/, "").trim();
|
|
2391
|
-
if (!t) return null;
|
|
2392
|
-
return { kind: "title", value: t };
|
|
2393
|
-
}
|
|
2394
|
-
function pathForPdfKey(key) {
|
|
2395
|
-
return join5(PDF_DIR, `${key.replace(/[^\w.-]+/g, "_")}.pdf`);
|
|
2396
|
-
}
|
|
2397
|
-
async function savePdfBuffer(key, buffer) {
|
|
2398
|
-
const path = pathForPdfKey(key);
|
|
2399
|
-
await mkdir4(dirname3(path), { recursive: true });
|
|
2400
|
-
await writeFile4(path, buffer);
|
|
2401
|
-
return { path, size: buffer.length, source: "library-import" };
|
|
2402
|
-
}
|
|
2403
|
-
async function importDir(dir, { autoResolve = true } = {}) {
|
|
2404
|
-
const target = resolve4(dir || "");
|
|
2405
|
-
if (!target) throw Object.assign(new Error("\u672A\u914D\u7F6E\u5BFC\u5165\u6587\u4EF6\u5939"), { code: "no_dir" });
|
|
2406
|
-
let files;
|
|
2407
|
-
try {
|
|
2408
|
-
files = await readdir2(target);
|
|
2409
|
-
} catch (e) {
|
|
2410
|
-
throw Object.assign(new Error(`\u65E0\u6CD5\u8BFB\u53D6\u6587\u4EF6\u5939 ${target}`), { code: "no_dir", cause: e });
|
|
2411
|
-
}
|
|
2412
|
-
const imported = [];
|
|
2413
|
-
const skipped = [];
|
|
2414
|
-
for (const name2 of files.filter((f) => extname(f).toLowerCase() === ".pdf")) {
|
|
2415
|
-
const abs = join5(target, name2);
|
|
2416
|
-
const seen = await getImportedFile(abs);
|
|
2417
|
-
let st;
|
|
2418
|
-
try {
|
|
2419
|
-
st = await stat2(abs);
|
|
2420
|
-
} catch {
|
|
2421
|
-
continue;
|
|
2422
|
-
}
|
|
2423
|
-
if (seen && seen.mtimeMs === st.mtimeMs) {
|
|
2424
|
-
skipped.push({ file: name2, reason: "already-imported" });
|
|
2425
|
-
continue;
|
|
2426
|
-
}
|
|
2427
|
-
let item;
|
|
2428
|
-
try {
|
|
2429
|
-
const hit = titleFromFilename(name2);
|
|
2430
|
-
if (!hit) {
|
|
2431
|
-
skipped.push({ file: name2, reason: "unparseable-name" });
|
|
2432
|
-
continue;
|
|
2433
|
-
}
|
|
2434
|
-
const provisional = buildItem({
|
|
2435
|
-
doi: hit.kind === "doi" ? hit.value : "",
|
|
2436
|
-
arxiv: hit.kind === "arxiv" ? hit.value : "",
|
|
2437
|
-
title: hit.kind === "title" ? hit.value : ""
|
|
2438
|
-
});
|
|
2439
|
-
if (!provisional.key) {
|
|
2440
|
-
skipped.push({ file: name2, reason: "no-identity" });
|
|
2441
|
-
continue;
|
|
2442
|
-
}
|
|
2443
|
-
const clash = await getItem(provisional.key);
|
|
2444
|
-
if (clash) {
|
|
2445
|
-
await addImportedFile(abs, st.mtimeMs);
|
|
2446
|
-
skipped.push({ file: name2, reason: "duplicate" });
|
|
2447
|
-
continue;
|
|
2448
|
-
}
|
|
2449
|
-
item = await putItem({
|
|
2450
|
-
...provisional,
|
|
2451
|
-
kind: hit.kind,
|
|
2452
|
-
rawValue: hit.value,
|
|
2453
|
-
display: hit.value,
|
|
2454
|
-
title: provisional.title || hit.value,
|
|
2455
|
-
state: "discovered",
|
|
2456
|
-
sourceFile: abs,
|
|
2457
|
-
createdAt: Date.now()
|
|
2458
|
-
});
|
|
2459
|
-
emitItem(item);
|
|
2460
|
-
} catch (e) {
|
|
2461
|
-
warn(`import ${name2} failed:`, e.message);
|
|
2462
|
-
skipped.push({ file: name2, reason: e.message });
|
|
2463
|
-
continue;
|
|
2464
|
-
}
|
|
2465
|
-
if (autoResolve) {
|
|
2466
|
-
try {
|
|
2467
|
-
const { resolveItem: resolveItem2 } = await Promise.resolve().then(() => (init_pipeline(), pipeline_exports));
|
|
2468
|
-
item = await resolveItem2(item.key);
|
|
2469
|
-
} catch (e) {
|
|
2470
|
-
warn(`auto-resolve ${name2} failed:`, e.message);
|
|
2471
|
-
}
|
|
2472
|
-
}
|
|
2473
|
-
await addImportedFile(abs, st.mtimeMs);
|
|
2474
|
-
imported.push(item?.key ?? provisionalKeyOf(item));
|
|
2475
|
-
}
|
|
2476
|
-
return { imported, skipped, dir: target };
|
|
2477
|
-
}
|
|
2478
|
-
function provisionalKeyOf(item) {
|
|
2479
|
-
return item?.key ?? "";
|
|
2480
|
-
}
|
|
2481
|
-
function zoteroItemToRecord(it) {
|
|
2482
|
-
const d = it?.data ?? it ?? {};
|
|
2483
|
-
const year = /^(\d{4})/.exec(String(d.date ?? ""))?.[1];
|
|
2484
|
-
return {
|
|
2485
|
-
source: "zotero",
|
|
2486
|
-
itemType: d.itemType ?? "journalArticle",
|
|
2487
|
-
title: d.title ?? "",
|
|
2488
|
-
authors: (d.creators ?? []).map((c) => ({
|
|
2489
|
-
creatorType: c.creatorType ?? "author",
|
|
2490
|
-
firstName: c.firstName ?? "",
|
|
2491
|
-
lastName: c.lastName ?? ""
|
|
2492
|
-
})),
|
|
2493
|
-
year: year ? Number(year) : null,
|
|
2494
|
-
container: d.publicationTitle ?? d.bookTitle ?? d.proceedingsTitle ?? "",
|
|
2495
|
-
publisher: d.publisher ?? "",
|
|
2496
|
-
volume: d.volume ?? "",
|
|
2497
|
-
issue: d.issue ?? "",
|
|
2498
|
-
pages: d.pages ?? "",
|
|
2499
|
-
doi: d.DOI ?? "",
|
|
2500
|
-
isbn: d.ISBN ?? "",
|
|
2501
|
-
url: d.url ?? "",
|
|
2502
|
-
abstract: d.abstractNote ?? ""
|
|
2503
|
-
};
|
|
2504
|
-
}
|
|
2505
|
-
async function importFromZotero({ limit = 50 } = {}) {
|
|
2506
|
-
const items = await searchItems("", { limit });
|
|
2507
|
-
const imported = [];
|
|
2508
|
-
const skipped = [];
|
|
2509
|
-
for (const it of items) {
|
|
2510
|
-
const rec = zoteroItemToRecord(it);
|
|
2511
|
-
if (!rec.title) {
|
|
2512
|
-
skipped.push({ reason: "no-title" });
|
|
2513
|
-
continue;
|
|
2514
|
-
}
|
|
2515
|
-
const provisional = buildItem(rec);
|
|
2516
|
-
if (!provisional.key) {
|
|
2517
|
-
skipped.push({ reason: "no-identity" });
|
|
2518
|
-
continue;
|
|
2519
|
-
}
|
|
2520
|
-
const existing = await getItem(provisional.key);
|
|
2521
|
-
if (existing) {
|
|
2522
|
-
skipped.push({ reason: "duplicate" });
|
|
2523
|
-
continue;
|
|
2524
|
-
}
|
|
2525
|
-
let pdf = null;
|
|
2526
|
-
try {
|
|
2527
|
-
const children = await getItemChildren(it.key);
|
|
2528
|
-
const attach = (children ?? []).find(
|
|
2529
|
-
(c) => c.itemType === "attachment" && /pdf/i.test(c.contentType ?? c.contentType ?? "")
|
|
2530
|
-
);
|
|
2531
|
-
if (attach?.key) {
|
|
2532
|
-
const { buffer } = await getFileBuffer(attach.key).catch(() => ({ buffer: null }));
|
|
2533
|
-
if (buffer?.length && buffer.subarray(0, 5).equals(PDF_MAGIC2)) {
|
|
2534
|
-
pdf = await savePdfBuffer(provisional.key, buffer);
|
|
2535
|
-
}
|
|
2536
|
-
}
|
|
2537
|
-
} catch (e) {
|
|
2538
|
-
warn(`attachment import failed for ${it.key}:`, e.message);
|
|
2539
|
-
}
|
|
2540
|
-
const item = await putItem({
|
|
2541
|
-
...provisional,
|
|
2542
|
-
record: rec,
|
|
2543
|
-
state: pdf ? "fetched" : "resolved",
|
|
2544
|
-
pdf,
|
|
2545
|
-
createdAt: Date.now()
|
|
2546
|
-
});
|
|
2547
|
-
emitItem(item);
|
|
2548
|
-
imported.push(item.key);
|
|
2549
|
-
}
|
|
2550
|
-
return { imported, skipped, count: imported.length };
|
|
2551
|
-
}
|
|
2552
|
-
function startWatcher() {
|
|
2553
|
-
if (watcherTimer) return () => stopWatcher();
|
|
2554
|
-
let stopping = false;
|
|
2555
|
-
const sweep = async () => {
|
|
2556
|
-
if (stopping) return;
|
|
2557
|
-
const config = await loadConfig();
|
|
2558
|
-
if (!config.watchImport || !config.importDir) return;
|
|
2559
|
-
try {
|
|
2560
|
-
const r = await importDir(config.importDir, { autoResolve: config.autoResolve !== false });
|
|
2561
|
-
if (r.imported.length) log(`folder watch imported ${r.imported.length} new file(s)`);
|
|
2562
|
-
} catch (e) {
|
|
2563
|
-
warn("folder watch sweep failed:", e.message);
|
|
2564
|
-
}
|
|
2565
|
-
};
|
|
2566
|
-
const initial = setTimeout(sweep, 5e3);
|
|
2567
|
-
initial.unref?.();
|
|
2568
|
-
const timer = setInterval(sweep, 3e4);
|
|
2569
|
-
timer.unref?.();
|
|
2570
|
-
watcherTimer = { timer, initial };
|
|
2571
|
-
return () => stopWatcher();
|
|
2572
|
-
}
|
|
2573
|
-
function stopWatcher() {
|
|
2574
|
-
if (!watcherTimer) return;
|
|
2575
|
-
clearInterval(watcherTimer.timer);
|
|
2576
|
-
clearTimeout(watcherTimer.initial);
|
|
2577
|
-
watcherTimer = null;
|
|
2578
|
-
}
|
|
2579
|
-
var PDF_MAGIC2, watcherTimer;
|
|
2580
|
-
var init_importer = __esm({
|
|
2581
|
-
"src/node/importer.js"() {
|
|
2582
|
-
init_config();
|
|
2583
|
-
init_db();
|
|
2584
|
-
init_sse();
|
|
2585
|
-
init_dedupe();
|
|
2586
|
-
init_metadata();
|
|
2587
|
-
init_local_api();
|
|
2588
|
-
init_log();
|
|
2589
|
-
PDF_MAGIC2 = Buffer.from("%PDF-", "latin1");
|
|
2590
|
-
watcherTimer = null;
|
|
2591
|
-
}
|
|
2592
|
-
});
|
|
2593
|
-
|
|
2594
2878
|
// src/node/index.js
|
|
2595
2879
|
init_config();
|
|
2596
2880
|
init_db();
|
|
@@ -2821,6 +3105,26 @@ async function handler(req, res) {
|
|
|
2821
3105
|
writeJson(res, 200, result);
|
|
2822
3106
|
return;
|
|
2823
3107
|
}
|
|
3108
|
+
if (head === "search" && methodOk(req, "POST")) {
|
|
3109
|
+
const body = await readJsonBody2(req);
|
|
3110
|
+
const candidates2 = await searchCandidates2(String(body?.q ?? ""), Number(body?.rows ?? 8));
|
|
3111
|
+
writeJson(res, 200, { candidates: candidates2 });
|
|
3112
|
+
return;
|
|
3113
|
+
}
|
|
3114
|
+
if (head === "add-candidate" && methodOk(req, "POST")) {
|
|
3115
|
+
const body = await readJsonBody2(req);
|
|
3116
|
+
const item = await addCandidate(body?.candidate ?? {});
|
|
3117
|
+
writeJson(res, 200, { item });
|
|
3118
|
+
return;
|
|
3119
|
+
}
|
|
3120
|
+
if (head === "drop" && methodOk(req, "POST")) {
|
|
3121
|
+
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
3122
|
+
const filename = url.searchParams.get("filename") || "dropped.pdf";
|
|
3123
|
+
const buffer = await readRawBody(req, 128 * 1024 * 1024);
|
|
3124
|
+
const item = await importDroppedPdf(buffer, { filename });
|
|
3125
|
+
writeJson(res, 200, { item });
|
|
3126
|
+
return;
|
|
3127
|
+
}
|
|
2824
3128
|
if (head === "import-zotero" && methodOk(req, "POST")) {
|
|
2825
3129
|
const body = await readJsonBody2(req);
|
|
2826
3130
|
const { importFromZotero: importFromZotero2 } = await Promise.resolve().then(() => (init_importer(), importer_exports));
|