@mantra-ai/core 0.1.0

package/dist/sources/medrxiv/client.js

@@ -0,0 +1,673 @@
+// services/sources/medrxiv/client.ts
+import { gunzipSync, inflateRawSync, inflateSync } from "node:zlib";
+import JSZip from "jszip";
+import { XMLParser } from "fast-xml-parser";
+import { fetchMedrxivSourceXmlViaBrowser, peekBrowserCookies } from "./browser";
+import { MEDRXIV_ACCEPT_LANGUAGE, MEDRXIV_ACCEPT_XML, MEDRXIV_BROWSER_UA, MEDRXIV_HTML_ACCEPT, looksLikeJatsXml, safePreview, } from "./shared";
+import { normPreprintCore, discoverPreprintRecord, parseVersionNumber, } from "../preprint-discovery";
+const DEBUG = process.env.DEBUG_MEDRXIV_FETCH === "1";
+const summarizeDetails = (details) => {
+    if (!details)
+        return "";
+    try {
+        const str = JSON.stringify(details);
+        return str.length > 500 ? `${str.slice(0, 500)}…` : str;
+    }
+    catch {
+        return String(details);
+    }
+};
+const logStrategyFailure = (result) => {
+    if (!DEBUG)
+        return;
+    const summary = summarizeDetails(result.details);
+    if (summary) {
+        console.log(`[medrxiv] ${result.name} failed: ${result.reason} ${summary}`);
+    }
+    else {
+        console.log(`[medrxiv] ${result.name} failed: ${result.reason}`);
+    }
+};
+function extractVersionFromString(value) {
+    if (!value)
+        return null;
+    const match = String(value)
+        .trim()
+        .match(/v(\d+)(?:[^0-9]|$)/i);
+    if (!match || !match[1])
+        return null;
+    const n = parseInt(match[1], 10);
+    return Number.isFinite(n) ? n : null;
+}
+const resolveVersion = (ctx) => {
+    if (ctx.inferredVersion && Number.isFinite(ctx.inferredVersion)) {
+        return Number(ctx.inferredVersion);
+    }
+    if (ctx.opts?.preferVersion && Number.isFinite(ctx.opts.preferVersion)) {
+        return Number(ctx.opts.preferVersion);
+    }
+    return null;
+};
+const directFetchStrategy = async (ctx) => {
+    const attempts = [];
+    const ua = ctx.opts?.UA || "drylab-reveng-api/1.0";
+    const browserUA = ua && /^Mozilla\//.test(ua) ? ua : MEDRXIV_BROWSER_UA;
+    const attempt = async (label, overrides) => {
+        try {
+            const res = await fetch(ctx.jatsUrl, {
+                headers: {
+                    Accept: MEDRXIV_ACCEPT_XML,
+                    "Accept-Language": MEDRXIV_ACCEPT_LANGUAGE,
+                    Referer: ctx.refererUrl,
+                    "User-Agent": browserUA,
+                    ...overrides,
+                },
+                redirect: "follow",
+            });
+            const contentType = res.headers.get("content-type") || "?";
+            DEBUG &&
+                console.log(`[medrxiv] ${label} status=`, res.status, "ct=", contentType);
+            if (!res.ok) {
+                attempts.push({ label, status: res.status, contentType });
+                return null;
+            }
+            const text = await res.text();
+            if (looksLikeJatsXml(text)) {
+                return { url: res.url || ctx.jatsUrl, xml: text };
+            }
+            attempts.push({
+                label,
+                status: res.status,
+                contentType,
+                reason: "non-jats",
+                preview: safePreview(text),
+            });
+            return null;
+        }
+        catch (err) {
+            attempts.push({ label, error: err?.message ?? String(err) });
+            return null;
+        }
+    };
+    const withCookies = async () => {
+        try {
+            const priming = await fetch(ctx.landingUrl, {
+                headers: {
+                    Accept: MEDRXIV_HTML_ACCEPT,
+                    "Accept-Language": MEDRXIV_ACCEPT_LANGUAGE,
+                    "User-Agent": browserUA,
+                },
+                redirect: "follow",
+            });
+            const contentType = priming.headers.get("content-type") || "?";
+            const raw = priming?.headers?.raw?.();
+            const setCookies = raw?.["set-cookie"] || [];
+            const cookieHeader = setCookies
+                .map((c) => String(c).split(";")[0])
+                .filter(Boolean)
+                .join("; ");
+            attempts.push({
+                label: "cookie-prime",
+                status: priming.status,
+                contentType,
+                cookies: setCookies.length,
+            });
+            DEBUG &&
+                console.log("[medrxiv] priming status:", priming.status, "content-type:", contentType, "cookies:", setCookies.length);
+            return cookieHeader || null;
+        }
+        catch (err) {
+            attempts.push({
+                label: "cookie-prime",
+                error: err?.message ?? String(err),
+            });
+            return null;
+        }
+    };
+    const attempt1 = await attempt("attempt#1");
+    if (attempt1) {
+        return {
+            success: true,
+            name: "direct",
+            data: { ...attempt1, version: resolveVersion(ctx) },
+        };
+    }
+    const attempt2 = await attempt("attempt#2");
+    if (attempt2) {
+        return {
+            success: true,
+            name: "direct",
+            data: { ...attempt2, version: resolveVersion(ctx) },
+        };
+    }
+    const cookies = await withCookies();
+    if (cookies) {
+        ctx.shared.cookies = cookies;
+        const attempt3 = await attempt("attempt#3", { Cookie: cookies });
+        if (attempt3) {
+            return {
+                success: true,
+                name: "direct",
+                data: { ...attempt3, version: resolveVersion(ctx) },
+            };
+        }
+    }
+    return {
+        success: false,
+        name: "direct",
+        reason: "blocked-or-non-jats",
+        details: { attempts },
+    };
+};
+const browserFetchStrategy = async (ctx) => {
+    const version = resolveVersion(ctx);
+    const payload = await fetchMedrxivSourceXmlViaBrowser({
+        coreId: ctx.coreId,
+        jatsUrl: ctx.jatsUrl,
+        inferredVersion: version,
+        refererUrl: ctx.refererUrl,
+        sourceOptions: ctx.opts,
+    });
+    const cookieHeader = payload?.cookies ?? peekBrowserCookies();
+    if (cookieHeader) {
+        ctx.shared.browserCookies = cookieHeader;
+    }
+    if (payload) {
+        return {
+            success: true,
+            name: "browser",
+            data: {
+                url: payload.url,
+                xml: payload.xml,
+                version: payload.version ?? version,
+            },
+        };
+    }
+    return {
+        success: false,
+        name: "browser",
+        reason: "browser-fetch-failed",
+    };
+};
+const proxyFetchStrategy = async (ctx) => {
+    const ua = ctx.opts?.UA || "drylab-reveng-api/1.0";
+    const browserUA = ua && /^Mozilla\//.test(ua) ? ua : MEDRXIV_BROWSER_UA;
+    const proxyUrl = `https://r.jina.ai/http://${ctx.jatsUrl.replace(/^https?:\/\//i, "")}`;
+    try {
+        DEBUG && console.log("[medrxiv] proxy fetch:", proxyUrl);
+        const res = await fetch(proxyUrl, {
+            headers: {
+                Accept: MEDRXIV_ACCEPT_XML,
+                "Accept-Language": MEDRXIV_ACCEPT_LANGUAGE,
+                "User-Agent": browserUA,
+            },
+        });
+        DEBUG && console.log("[medrxiv] proxy status:", res.status);
+        if (!res.ok) {
+            return {
+                success: false,
+                name: "proxy",
+                reason: `status-${res.status}`,
+            };
+        }
+        const text = await res.text();
+        const head = text.slice(0, 512);
+        DEBUG &&
+            console.log("[medrxiv] proxy head:", head.replace(/\s+/g, " ").slice(0, 200));
+        if (!looksLikeJatsXml(text)) {
+            return {
+                success: false,
+                name: "proxy",
+                reason: "non-jats",
+                details: { preview: safePreview(text) },
+            };
+        }
+        return {
+            success: true,
+            name: "proxy",
+            data: {
+                url: ctx.jatsUrl,
+                xml: text,
+                version: resolveVersion(ctx),
+            },
+        };
+    }
+    catch (err) {
+        return {
+            success: false,
+            name: "proxy",
+            reason: err?.message ?? String(err),
+        };
+    }
+};
+const connectFetchStrategy = async (ctx) => {
+    const version = resolveVersion(ctx);
+    if (!version) {
+        return {
+            success: false,
+            name: "connect",
+            reason: "missing-version",
+        };
+    }
+    const fallback = await fetchMedrxivFromConnect(ctx.coreId, version, ctx.opts, ctx.shared.browserCookies || ctx.shared.cookies);
+    if (fallback) {
+        return {
+            success: true,
+            name: "connect",
+            data: fallback,
+        };
+    }
+    return {
+        success: false,
+        name: "connect",
+        reason: "connect-fallback-null",
+    };
+};
+/** Return the versioned .source.xml record for the best-matching version (or null) */
+export async function buildMedrxivLatestSourceXmlUrlViaApi(idOrDoi, opts) {
+    return discoverPreprintRecord(idOrDoi, "medrxiv", {
+        UA: opts?.UA,
+        preferVersion: opts?.preferVersion,
+        debug: DEBUG,
+    });
+}
+/** Fetch the JATS XML content via the API-discovered URL */
+export async function fetchMedrxivSourceXmlViaApi(idOrDoi, opts) {
+    const record = await buildMedrxivLatestSourceXmlUrlViaApi(idOrDoi, opts);
+    const core = normPreprintCore(idOrDoi, "medrxiv");
+    const inferredVersion = record?.version ??
+        opts?.preferVersion ??
+        extractVersionFromString(record?.url);
+    const jatsUrl = record?.url;
+    if (!jatsUrl) {
+        return await fetchMedrxivFromConnect(core, inferredVersion, opts);
+    }
+    const landingUrl = `https://www.medrxiv.org/content/10.1101/${core}`;
+    const versionSuffix = (() => {
+        if (inferredVersion && Number.isFinite(Number(inferredVersion))) {
+            return `v${Number(inferredVersion)}`;
+        }
+        const match = record?.url && record.url.match(/v(\d+)/i);
+        return match ? match[0].toLowerCase() : "";
+    })();
+    const refererUrl = versionSuffix
+        ? `${landingUrl}${versionSuffix}`
+        : landingUrl;
+    DEBUG && console.log("[medrxiv] trying jats url:", jatsUrl);
+    const ctx = {
+        coreId: core,
+        jatsUrl,
+        landingUrl,
+        refererUrl,
+        inferredVersion: inferredVersion ?? null,
+        opts,
+        shared: {},
+    };
+    const strategies = [
+        directFetchStrategy,
+        browserFetchStrategy,
+        proxyFetchStrategy,
+        connectFetchStrategy,
+    ];
+    for (const strategy of strategies) {
+        const result = await strategy(ctx);
+        if (result.success) {
+            DEBUG && console.log(`[medrxiv] ${result.name} strategy succeeded`);
+            return result.data;
+        }
+        logStrategyFailure(result);
+    }
+    DEBUG && console.log("[medrxiv] all strategies failed");
+    return null;
+}
+async function fetchMedrxivFromConnect(core, version, opts, cookieHeader) {
+    if (!version || !Number.isFinite(version))
+        return null;
+    const defaultUA = opts?.UA || "drylab-reveng-api/1.0";
+    const browserUA = defaultUA && /^Mozilla\//.test(defaultUA) ? defaultUA : MEDRXIV_BROWSER_UA;
+    const suffix = `v${version}`;
+    const baseFile = `${core}${suffix}`;
+    const candidates = [
+        `https://connect.medrxiv.org/relate/content/181/files/${baseFile}.source.xml`,
+        `https://connect.medrxiv.org/relate/content/181/files/${baseFile}.source.xml?download=1`,
+        `https://connect.medrxiv.org/relate/content/181/files/${baseFile}.full.xml`,
+        `https://connect.medrxiv.org/relate/content/181/files/${baseFile}.full.xml?download=1`,
+        `https://connect.medrxiv.org/relate/feed/181/files/${baseFile}.source.xml`,
+        `https://connect.medrxiv.org/relate/feed/181/files/${baseFile}.source.xml?download=1`,
+        `https://connect.medrxiv.org/relate/feed/181/files/${baseFile}.full.xml`,
+        `https://connect.medrxiv.org/relate/feed/181/files/${baseFile}.full.xml?download=1`,
+    ];
+    const queue = [...candidates];
+    const tried = new Set();
+    let currentCookieHeader = cookieHeader ?? null;
+    const maxAttempts = 20;
+    while (queue.length && tried.size < maxAttempts) {
+        const url = queue.shift();
+        if (tried.has(url))
+            continue;
+        tried.add(url);
+        try {
+            DEBUG && console.log("[medrxiv] connect fallback url:", url);
+            const res = await fetch(url, {
+                headers: {
+                    Accept: MEDRXIV_ACCEPT_XML,
+                    "User-Agent": browserUA,
+                    Referer: `https://www.medrxiv.org/content/10.1101/${core}${suffix}`,
+                    "Accept-Language": MEDRXIV_ACCEPT_LANGUAGE,
+                    ...(currentCookieHeader ? { Cookie: currentCookieHeader } : {}),
+                },
+                redirect: "follow",
+            });
+            DEBUG &&
+                console.log("[medrxiv] connect status:", res.status, res.headers.get("content-type") || "?");
+            if (!res.ok)
+                continue;
+            const rawSetCookie = res?.headers?.raw?.()?.["set-cookie"] || [];
+            if (Array.isArray(rawSetCookie) && rawSetCookie.length) {
+                const merged = new Map();
+                if (currentCookieHeader) {
+                    currentCookieHeader.split(/;\s*/).forEach((pair) => {
+                        const [name, value] = pair.split("=");
+                        if (name && value !== undefined)
+                            merged.set(name.trim(), value);
+                    });
+                }
+                for (const setCookie of rawSetCookie) {
+                    const [pair] = String(setCookie).split(";");
+                    if (!pair)
+                        continue;
+                    const [name, value] = pair.split("=");
+                    if (name && value !== undefined)
+                        merged.set(name.trim(), value);
+                }
+                currentCookieHeader = Array.from(merged.entries())
+                    .map(([name, value]) => `${name}=${value}`)
+                    .join("; ");
+            }
+            const rawBuffer = Buffer.from(await res.arrayBuffer());
+            const decoded = await decodeConnectResponse(rawBuffer, res.headers);
+            if (!decoded) {
+                const asText = bufferToUtf8(rawBuffer);
+                if (asText) {
+                    if (DEBUG) {
+                        console.log("[medrxiv] connect non-jats response", safePreview(asText));
+                    }
+                    const extra = collectFollowupUrlsFromText(asText, url);
+                    if (DEBUG && extra.length) {
+                        console.log("[medrxiv] connect discovered urls", extra);
+                    }
+                    for (const next of extra) {
+                        if (!tried.has(next))
+                            queue.push(next);
+                    }
+                }
+                continue;
+            }
+            if (!looksLikeJatsXml(decoded)) {
+                if (DEBUG) {
+                    console.log("[medrxiv] connect non-jats decoded", safePreview(decoded));
+                }
+                const extra = collectFollowupUrlsFromText(decoded, url);
+                if (DEBUG && extra.length) {
+                    console.log("[medrxiv] connect discovered urls", extra);
+                }
+                for (const next of extra) {
+                    if (!tried.has(next))
+                        queue.push(next);
+                }
+                continue;
+            }
+            return { url, xml: decoded, version };
+        }
+        catch (err) {
+            DEBUG &&
+                console.log("[medrxiv] connect fetch error", err?.message ?? String(err));
+        }
+    }
+    return null;
+}
+async function decodeConnectResponse(rawBuffer, headers) {
+    if (!rawBuffer || !rawBuffer.length)
+        return null;
+    const attempts = [];
+    const encoding = (headers.get("content-encoding") || "").toLowerCase().trim();
+    if (encoding.includes("gzip")) {
+        attempts.push(tryGunzip(rawBuffer));
+    }
+    else if (encoding.includes("deflate")) {
+        attempts.push(tryInflate(rawBuffer));
+    }
+    // Always attempt common encodings regardless of explicit headers
+    attempts.push(tryGunzip(rawBuffer));
+    attempts.push(tryInflate(rawBuffer));
+    attempts.push(rawBuffer);
+    for (const candidate of attempts) {
+        if (!candidate || !candidate.length)
+            continue;
+        const text = bufferToUtf8(candidate);
+        if (DEBUG) {
+            console.log("[medrxiv] connect decode candidate", {
+                size: candidate.length,
+                looksLike: text ? looksLikeJatsXml(text) : false,
+                preview: text ? safePreview(text, 200) : null,
+            });
+        }
+        if (text && looksLikeJatsXml(text))
+            return text;
+    }
+    if (isZipArchive(rawBuffer)) {
+        try {
+            const zip = await JSZip.loadAsync(rawBuffer);
+            DEBUG &&
+                console.log("[medrxiv] connect decode zip entries", Object.keys(zip.files));
+            const preferred = Object.values(zip.files)
+                .filter((f) => !f.dir)
+                .sort((a, b) => scoreZipName(a.name) - scoreZipName(b.name));
+            for (const file of preferred) {
+                const txt = await file.async("string");
+                const cleaned = txt.replace(/^\uFEFF/, "");
+                DEBUG &&
+                    console.log("[medrxiv] connect decode zip entry", file.name, cleaned.slice(0, 200).replace(/\s+/g, " "));
+                if (cleaned && looksLikeJatsXml(cleaned))
+                    return cleaned;
+            }
+        }
+        catch { }
+    }
+    return null;
+}
+function tryGunzip(buf) {
+    if (!buf || !buf.length)
+        return null;
+    try {
+        return gunzipSync(buf);
+    }
+    catch { }
+    // Some servers return gzip payloads without headers
+    if (buf.length > 2 && buf[0] === 0x1f && buf[1] === 0x8b) {
+        try {
+            return gunzipSync(buf);
+        }
+        catch { }
+    }
+    return null;
+}
+function tryInflate(buf) {
+    if (!buf || !buf.length)
+        return null;
+    try {
+        return inflateSync(buf);
+    }
+    catch { }
+    try {
+        return inflateRawSync(buf);
+    }
+    catch { }
+    return null;
+}
+function bufferToUtf8(buf) {
+    if (!buf || !buf.length)
+        return null;
+    const text = buf.toString("utf-8");
+    const stripped = text.replace(/^\uFEFF/, "");
+    return stripped.length ? stripped : null;
+}
+function collectFollowupUrlsFromText(text, baseUrl) {
+    if (!text.trim())
+        return [];
+    if (/<html/i.test(text))
+        return extractHtmlDownloadUrls(text, baseUrl);
+    if (/<rss\b/i.test(text) || /<feed\b/i.test(text))
+        return extractFeedDownloadUrls(text, baseUrl);
+    return extractGenericXmlLinks(text, baseUrl);
+}
+function extractHtmlDownloadUrls(html, baseUrl) {
+    const urls = new Set();
+    const normalized = html.replace(/&amp;/gi, "&");
+    const metaMatch = normalized.match(/http-equiv=["']refresh["'][^>]*content=["'][^"']*url=([^"' >]+)/i);
+    if (metaMatch && metaMatch[1]) {
+        urls.add(resolveRelative(metaMatch[1], baseUrl));
+    }
+    const hrefRegex = /href\s*=\s*["']([^"']+\.(?:source|full)\.xml[^"']*)["']/gi;
+    let hrefMatch;
+    while ((hrefMatch = hrefRegex.exec(normalized))) {
+        urls.add(resolveRelative(hrefMatch[1], baseUrl));
+    }
+    const locationRegex = /location\s*\.\s*(?:href|replace)\s*=\s*["']([^"']+\.xml[^"']*)["']/gi;
+    let locMatch;
+    while ((locMatch = locationRegex.exec(normalized))) {
+        urls.add(resolveRelative(locMatch[1], baseUrl));
+    }
+    return Array.from(urls);
+}
+const feedParser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: "@_",
+    textNodeName: "#text",
+});
+function extractFeedDownloadUrls(xmlText, baseUrl) {
+    const urls = new Set();
+    try {
+        const doc = feedParser.parse(xmlText);
+        const addCandidate = (candidate) => {
+            if (!candidate)
+                return;
+            const str = String(candidate).trim();
+            if (!str)
+                return;
+            if (/\.xml($|\?|#)/i.test(str))
+                urls.add(resolveRelative(str, baseUrl));
+        };
+        if (doc.feed) {
+            const entries = Array.isArray(doc.feed.entry)
+                ? doc.feed.entry
+                : doc.feed.entry
+                    ? [doc.feed.entry]
+                    : [];
+            for (const entry of entries) {
+                const links = entry?.link;
+                if (Array.isArray(links)) {
+                    for (const link of links)
+                        addCandidate(link?.["@_href"] || link?.href);
+                }
+                else if (links) {
+                    addCandidate(links?.["@_href"] || links?.href);
+                }
+                addCandidate(entry?.content?.["@_src"] || entry?.content?.src);
+                if (typeof entry?.content === "string")
+                    addCandidate(entry.content);
+                if (typeof entry?.summary === "string")
+                    addCandidate(entry.summary);
+                addCandidate(entry?.id);
+            }
+            addCandidate(doc.feed?.link?.["@_href"] || doc.feed?.link);
+        }
+        if (doc.rss?.channel) {
+            const channel = doc.rss.channel;
+            const items = Array.isArray(channel.item)
+                ? channel.item
+                : channel.item
+                    ? [channel.item]
+                    : [];
+            for (const item of items) {
+                const enc = item?.enclosure;
+                if (Array.isArray(enc)) {
+                    for (const e of enc)
+                        addCandidate(e?.["@_url"] || e?.url);
+                }
+                else if (enc)
+                    addCandidate(enc?.["@_url"] || enc?.url);
+                addCandidate(item?.link);
+                addCandidate(item?.guid);
+                if (typeof item?.description === "string")
+                    addCandidate(item.description);
+            }
+            addCandidate(channel?.link);
+        }
+    }
+    catch { }
+    if (!urls.size) {
+        const fallback = extractGenericXmlLinks(xmlText, baseUrl);
+        for (const u of fallback)
+            urls.add(u);
+    }
+    return Array.from(urls);
+}
+function extractGenericXmlLinks(text, baseUrl) {
+    const urls = new Set();
+    const regex = /https?:\/\/[^\s"'>]+\.xml[^\s"'>]*/gi;
+    let match;
+    while ((match = regex.exec(text))) {
+        urls.add(resolveRelative(match[0], baseUrl));
+    }
+    return Array.from(urls);
+}
+function resolveRelative(candidate, baseUrl) {
+    try {
+        const trimmed = candidate.trim();
+        if (!trimmed)
+            return baseUrl;
+        const absolute = new URL(trimmed, baseUrl);
+        return absolute.toString();
+    }
+    catch {
+        return baseUrl;
+    }
+}
+function isZipArchive(buf) {
+    return (buf.length > 4 &&
+        buf[0] === 0x50 &&
+        buf[1] === 0x4b &&
+        buf[2] === 0x03 &&
+        buf[3] === 0x04);
+}
+function scoreZipName(name) {
+    const lower = name.toLowerCase();
+    if (lower.endsWith(".source.xml"))
+        return 0;
+    if (lower.endsWith(".full.xml"))
+        return 1;
+    if (lower.endsWith(".xml"))
+        return 2;
+    return 10;
+}
+export const __medrxivTestExports = {
+    resolveVersion,
+    collectFollowupUrlsFromText,
+};
+/** Get the latest version number for a medRxiv core/DOI (null if unknown). */
+export async function fetchLatestMedrxivVersionNumber(idOrDoi, opts) {
+    const rec = await buildMedrxivLatestSourceXmlUrlViaApi(idOrDoi, opts);
+    if (!rec)
+        return null;
+    if (typeof rec.version === "number" && Number.isFinite(rec.version))
+        return rec.version;
+    // Fallback: try to parse ...vN from the returned URL
+    const m = rec.url && rec.url.match(/v(\d+)(?:\D|$)/i);
+    if (m && m[1]) {
+        const n = parseInt(m[1], 10);
+        return Number.isFinite(n) ? n : null;
+    }
+    return null;
+}
+//# sourceMappingURL=client.js.map