@mantra-ai/core 0.1.0

package/dist/normalization/pdf/normalize.js

@@ -0,0 +1,321 @@
+import { createHash } from "node:crypto";
+import { generateContentWithPdf } from "../../ai/google/client";
+import { NormalizedDocSchema, } from "../normalized-doc-schema";
+import { normalizeDOI, normalizePMCID } from "../../works/id/normalize-ids";
+import { applyNormalizedCompleteIds } from "../../works/id/resolveIds/assign";
+import { PipelineError, PIPELINE_ERROR_CODES, } from "../../works/errors";
+import { PDF_EXTRACTION_PROMPT, PDF_EXTRACTION_SYSTEM } from "./prompt";
+function extractFirstJson(text, truncated = false) {
+    const trimmed = text.trim();
+    try {
+        return JSON.parse(trimmed);
+    }
+    catch { }
+    // Strip markdown fencing if present
+    const unfenced = trimmed
+        .replace(/^```(?:json)?\s*/i, "")
+        .replace(/\s*```$/i, "")
+        .trim();
+    try {
+        return JSON.parse(unfenced);
+    }
+    catch { }
+    // Find the first { and try to parse from there
+    const objStart = unfenced.indexOf("{");
+    if (objStart === -1)
+        return null;
+    const candidate = unfenced.slice(objStart);
+    try {
+        return JSON.parse(candidate);
+    }
+    catch { }
+    // If truncated, attempt to repair by closing open structures
+    if (truncated) {
+        const repaired = repairTruncatedJson(candidate);
+        if (repaired)
+            return repaired;
+    }
+    return null;
+}
+/**
+ * Attempt to repair JSON truncated mid-stream by closing open structures
+ * in the correct nesting order using a stack.
+ */
+function repairTruncatedJson(text) {
+    // Strip trailing partial value from the end (work backwards to avoid
+    // expensive regex on 200KB+ strings)
+    let end = text.length;
+    // Walk back past whitespace
+    while (end > 0 && " \t\n\r".includes(text[end - 1]))
+        end--;
+    // If we're mid-string (odd number of unescaped quotes at tail), trim to last clean point
+    // If trailing comma, strip it
+    let cleaned = text.slice(0, end);
+    cleaned = cleaned.replace(/,\s*$/, "");
+    // Track nesting with a stack to close in correct order
+    const stack = [];
+    let inString = false;
+    let escaped = false;
+    for (const ch of cleaned) {
+        if (escaped) {
+            escaped = false;
+            continue;
+        }
+        if (ch === "\\") {
+            escaped = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString)
+            continue;
+        if (ch === "{")
+            stack.push("}");
+        else if (ch === "[")
+            stack.push("]");
+        else if (ch === "}" || ch === "]")
+            stack.pop();
+    }
+    // If we ended inside a string, close it
+    if (inString)
+        cleaned += '"';
+    // If the last complete token was a key with no value (e.g. `"surname":`),
+    // add a null so the object entry is valid
+    if (/:\s*$/.test(cleaned))
+        cleaned += "null";
+    // Close open structures in reverse nesting order
+    while (stack.length > 0)
+        cleaned += stack.pop();
+    try {
+        return JSON.parse(cleaned);
+    }
+    catch { }
+    // If that didn't work, try progressively trimming trailing partial entries
+    // by removing the last line repeatedly (up to 20 lines)
+    let fallback = cleaned;
+    for (let i = 0; i < 20; i++) {
+        const lastNewline = fallback.lastIndexOf("\n");
+        if (lastNewline === -1)
+            break;
+        fallback = fallback.slice(0, lastNewline).replace(/,\s*$/, "");
+        // Recount stack
+        const st = [];
+        let inStr = false;
+        let esc = false;
+        for (const ch of fallback) {
+            if (esc) {
+                esc = false;
+                continue;
+            }
+            if (ch === "\\") {
+                esc = true;
+                continue;
+            }
+            if (ch === '"') {
+                inStr = !inStr;
+                continue;
+            }
+            if (inStr)
+                continue;
+            if (ch === "{")
+                st.push("}");
+            else if (ch === "[")
+                st.push("]");
+            else if (ch === "}" || ch === "]")
+                st.pop();
+        }
+        if (inStr)
+            fallback += '"';
+        if (/:\s*$/.test(fallback))
+            fallback += "null";
+        let attempt = fallback;
+        while (st.length > 0)
+            attempt += st.pop();
+        try {
+            return JSON.parse(attempt);
+        }
+        catch { }
+    }
+    return null;
+}
+/**
+ * Recursively strip null values from a parsed JSON object.
+ * Gemini often returns `null` for missing fields instead of omitting the key,
+ * which causes Zod `.optional()` fields to fail validation.
+ */
+function stripNulls(obj) {
+    if (obj === null)
+        return undefined;
+    if (Array.isArray(obj))
+        return obj.map(stripNulls);
+    if (typeof obj === "object") {
+        const out = {};
+        for (const [k, v] of Object.entries(obj)) {
+            if (v !== null)
+                out[k] = stripNulls(v);
+        }
+        return out;
+    }
+    return obj;
+}
+const RETRY_DELAY_MS = 500;
+const PRIMARY_MODEL = "gemini-2.5-flash-lite";
+const FALLBACK_CHAIN = [
+    { strategy: "inline", model: PRIMARY_MODEL, label: "inline/2.5-flash-lite" },
+    { strategy: "inline", model: PRIMARY_MODEL, label: "inline/2.5-flash-lite/no-json-mime", jsonMime: false },
+    { strategy: "file_upload", model: PRIMARY_MODEL, label: "file_upload/2.5-flash-lite" },
+];
+export async function normalizePdfToDoc(input) {
+    const { pdfBuffer, completeIds, logger, cid } = input;
+    const pdfBase64 = pdfBuffer.toString("base64");
+    const pdfSha256 = createHash("sha256").update(pdfBuffer).digest("hex");
+    let lastError;
+    let lastCode;
+    for (let i = 0; i < FALLBACK_CHAIN.length; i++) {
+        const step = FALLBACK_CHAIN[i];
+        const res = await generateContentWithPdf({
+            pdfBase64,
+            pdfBuffer,
+            prompt: PDF_EXTRACTION_PROMPT,
+            systemInstruction: PDF_EXTRACTION_SYSTEM,
+            model: step.model,
+            strategy: step.strategy,
+            temperature: 0.1,
+            maxOutputTokens: 65536,
+            ...(step.jsonMime !== false && { responseMimeType: "application/json" }),
+        });
+        if (!res.ok) {
+            lastError = res.error;
+            lastCode = res.code;
+            logger?.warn?.({
+                correlationId: cid,
+                step: step.label,
+                stepIndex: i,
+                code: res.code,
+                finishReason: res.meta?.finishReason,
+                error: res.error,
+            }, "pdf.normalize.gemini_error");
+            // For RECITATION, escalate to next fallback step immediately
+            if (res.code === "recitation") {
+                if (i < FALLBACK_CHAIN.length - 1) {
+                    await new Promise((r) => setTimeout(r, RETRY_DELAY_MS));
+                    continue;
+                }
+                break;
+            }
+            // For other errors, don't retry (API errors, safety blocks, etc.)
+            break;
+        }
+        const rawText = res.text;
+        const isTruncated = res.meta.finishReason === "MAX_TOKENS";
+        if (isTruncated) {
+            logger?.warn?.({ correlationId: cid, step: step.label, textLength: rawText.length }, "pdf.normalize.truncated_response");
+        }
+        const parsed = extractFirstJson(rawText, isTruncated);
+        if (!parsed) {
+            lastError = "Could not extract JSON from Gemini response";
+            lastCode = "json_extraction_failed";
+            logger?.warn?.({
+                correlationId: cid,
+                step: step.label,
+                stepIndex: i,
+                rawTextLength: rawText.length,
+            }, "pdf.normalize.json_extraction_failed");
+            // JSON parse failure is not a RECITATION issue — don't escalate strategies
+            break;
+        }
+        const validated = NormalizedDocSchema.safeParse(stripNulls(parsed));
+        if (!validated.success) {
+            lastError = `Schema validation failed: ${JSON.stringify(validated.error.flatten())}`;
+            lastCode = "schema_validation_failed";
+            logger?.warn?.({
+                correlationId: cid,
+                step: step.label,
+                stepIndex: i,
+                errors: validated.error.flatten(),
+            }, "pdf.normalize.schema_validation_failed");
+            // Schema failure is not a RECITATION issue — don't escalate strategies
+            break;
+        }
+        const normalizedDoc = validated.data;
+        const meta = normalizedDoc.metadata;
+        // Merge IDs from extracted metadata into completeIds (non-destructive)
+        const metaIds = meta?.ids ?? {};
+        const mergeIfEmpty = (key, value) => {
+            if (!value)
+                return;
+            if (completeIds[key])
+                return;
+            completeIds[key] = value;
+        };
+        if (metaIds.doi)
+            mergeIfEmpty("doi", normalizeDOI(metaIds.doi));
+        if (metaIds.pmcid)
+            mergeIfEmpty("pmcid", normalizePMCID(metaIds.pmcid));
+        applyNormalizedCompleteIds(completeIds);
+        // Extract convenience fields
+        const title = meta?.title ?? undefined;
+        const abstractText = meta?.abstract?.paragraphs?.join(" ") ?? undefined;
+        const licenseFromMeta = meta?.license?.href ?? meta?.license?.type ?? null;
+        const versionLabel = completeIds.version ?? undefined;
+        logger?.info?.({
+            correlationId: cid,
+            source: "pdf-gemini",
+            step: step.label,
+            haveTitle: Boolean(title),
+            haveAbstract: Boolean(abstractText),
+            sectionCount: normalizedDoc.sections?.length ?? 0,
+            blockCount: normalizedDoc.blocks?.length ?? 0,
+            citationCount: normalizedDoc.citations?.length ?? 0,
+            versionLabel,
+        }, "pdf.normalize.success");
+        return {
+            isPdf: true,
+            versionLabel,
+            normalizedJson: normalizedDoc,
+            metadata: meta || null,
+            schemaVersion: normalizedDoc.schemaVersion ?? null,
+            title,
+            abstractText,
+            licenseFromMeta,
+            license: null,
+            source: "pdf-gemini",
+            sourceUrl: null,
+            rawXml: null,
+            pdfSha256,
+            blocks: normalizedDoc.blocks,
+            sections: normalizedDoc.sections,
+            citations: normalizedDoc.citations,
+            assets: normalizedDoc.assets,
+        };
+    }
+    const isRecitationExhausted = lastCode === "recitation";
+    throw new PipelineError(isRecitationExhausted
+        ? PIPELINE_ERROR_CODES.UPSTREAM_FAILURE
+        : PIPELINE_ERROR_CODES.INTERNAL_ERROR, isRecitationExhausted
+        ? `PDF normalization failed: all strategies exhausted due to Gemini RECITATION block. The model refused to extract content from this PDF across ${FALLBACK_CHAIN.length} fallback strategies.`
+        : `PDF normalization failed: ${lastError}`, {
+        failureClass: lastCode ?? "unknown",
+        strategiesAttempted: FALLBACK_CHAIN.length,
+    });
+}
+export function buildPdfRawEnvelope(pdfBuffer, pdfSha256) {
+    return {
+        xml: "", // no XML for PDF source
+        size_bytes: pdfBuffer.length,
+        hash_sha256: pdfSha256,
+        content_type: "application/pdf",
+        encoding: "binary",
+        source: { name: "pdf-upload" },
+        normalization: {
+            schema_version: "v1",
+            normalizer_version: "pdf-gemini-1.0",
+            input_hash: pdfSha256,
+            created_at: new Date().toISOString(),
+            warnings: [],
+        },
+    };
+}
+//# sourceMappingURL=normalize.js.map

package/dist/normalization/pdf/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../../src/normalization/pdf/normalize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,EACL,mBAAmB,GAEpB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AA6BxE,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,KAAK;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,oCAAoC;IACpC,MAAM,QAAQ,GAAG,OAAO;SACrB,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;SAChC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,IAAI,EAAE,CAAC;IACV,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,+CAA+C;IAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,QAAQ,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,6DAA6D;IAC7D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;IAChC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,qEAAqE;IACrE,qCAAqC;IACrC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,OAAO,GAAG,GAAG,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IAE5D,yFAAyF;IACzF,8BAA8B;IAC9B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEvC,uDAAuD;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,OAAO,EAAE,CAAC;YAAC,OAAO,GAAG,KAAK,CAAC;YAAC,SAAS;QAAC,CAAC;QAC3C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9C,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,QAAQ,GAAG,CAAC,QAAQ,CAAC;YAAC,SAAS;QAAC,CAAC;QACnD,IAAI,QAAQ;YAAE,SAAS;QACvB,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAC3B,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAChC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,CAAC,GAAG,EAAE,CAAC;IACjD,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ;QAAE,OAAO,IAAI,GAAG,CAAC;IAE7B,0EAA0E;IAC1E,0CAA0C;IAC1C,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,MAAM,CAAC;IAE7C,iDAAiD;IACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;IAEhD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,QAAQ,GAAG,OAAO,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,WAAW,KAAK,CAAC,CAAC;YAAE,MAAM;QAC9B,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAE/D,gBAAgB;QAChB,MAAM,EAAE,GAAa,EAAE,CAAC;QACxB,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,GAAG,EAAE,CAAC;gBAAC,GAAG,GAAG,KAAK,CAAC;gBAAC,SAAS;YAAC,CAAC;YACnC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBAAC,GAAG,GAAG,IAAI,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC1C,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,KAAK,GAAG,CAAC,KAAK,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC7C,IAAI,KAAK;gBAAE,SAAS;YACpB,IAAI,EAAE,KAAK,GAAG;gBAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACxB,IAAI,EAAE,KAAK,GAAG;gBAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBAC7B,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;gBAAE,EAAE,CAAC,GAAG,EAAE,CAAC;QAC9C,CAAC;QACD,IAAI,KAAK;YAAE,QAAQ,IAAI,GAAG,CAAC;QAC3B,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,QAAQ,IAAI,MAAM,CAAC;QAC/C,IAAI,OAAO,GAAG,QAAQ,CAAC;QACvB,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,GAAY;IAC9B,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACnC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;YACpE,IAAI,CAAC,KAAK,IAAI;gBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,cAAc,GAAG,GAAG,CAAC;AAa3B,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAE9C,MAAM,cAAc,GAAmB;IACrC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,uBAAuB,EAAE;IAC5E,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,oCAAoC,EAAE,QAAQ,EAAE,KAAK,EAAE;IAC1G,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,4BAA4B,EAAE;CACvF,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAwB;IAExB,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IAEtD,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEvE,IAAI,SAA6B,CAAC;IAClC,IAAI,QAA4B,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAE,CAAC;QAEhC,MAAM,GAAG,GAAG,MAAM,sBAAsB,CAAC;YACvC,SAAS;YACT,SAAS;YACT,MAAM,EAAE,qBAAqB;YAC7B,iBAAiB,EAAE,qBAAqB;YACxC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,GAAG;YAChB,eAAe,EAAE,KAAK;YACtB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,CAAC;SACzE,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;YACtB,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC;YACpB,MAAM,EAAE,IAAI,EAAE,CACZ;gBACE,aAAa,EAAE,GAAG;gBAClB,IAAI,EAAE,IAAI,CAAC,KAAK;gBAChB,SAAS,EAAE,CAAC;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,YAAY,EAAE,GAAG,CAAC,IAAI,EAAE,YAAY;gBACpC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,EACD,4BAA4B,CAC7B,CAAC;YAEF,6DAA6D;YAC7D,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC9B,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;oBACxD,SAAS;gBACX,CAAC;gBACD,MAAM;YACR,CAAC;YAED,kEAAkE;YAClE,MAAM;QACR,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC;QACzB,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,YAAY,KAAK,YAAY,CAAC;QAC3D,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,EAAE,IAAI,EAAE,CACZ,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,EACpE,kCAAkC,CACnC,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS,GAAG,6CAA6C,CAAC;YAC1D,QAAQ,GAAG,wBAAwB,CAAC;YACpC,MAAM,EAAE,IAAI,EAAE,CACZ;gBACE,aAAa,EAAE,GAAG;gBAClB,IAAI,EAAE,IAAI,CAAC,KAAK;gBAChB,SAAS,EAAE,CAAC;gBACZ,aAAa,EAAE,OAAO,CAAC,MAAM;aAC9B,EACD,sCAAsC,CACvC,CAAC;YACF,2EAA2E;YAC3E,MAAM;QACR,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACvB,SAAS,GAAG,6BAA6B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACrF,QAAQ,GAAG,0BAA0B,CAAC;YACtC,MAAM,EAAE,IAAI,EAAE,CACZ;gBACE,aAAa,EAAE,GAAG;gBAClB,IAAI,EAAE,IAAI,CAAC,KAAK;gBAChB,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;aAClC,EACD,wCAAwC,CACzC,CAAC;YACF,uEAAuE;YACvE,MAAM;QACR,CAAC;QAED,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC;QACrC,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,CAAC;QAEpC,uEAAuE;QACvE,MAAM,OAAO,GAAG,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;QAChC,MAAM,YAAY,GAAG,CAAC,GAAsB,EAAE,KAAqB,EAAE,EAAE;YACrE,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,IAAI,WAAW,CAAC,GAAG,CAAC;gBAAE,OAAO;YAC5B,WAAmB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACpC,CAAC,CAAC;QAEF,IAAI,OAAO,CAAC,GAAG;YAAE,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,OAAO,CAAC,KAAK;YAAE,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,0BAA0B,CAAC,WAAW,CAAC,CAAC;QAExC,6BAA6B;QAC7B,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,SAAS,CAAC;QACvC,MAAM,YAAY,GAAG,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC;QACxE,MAAM,eAAe,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC;QAC3E,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,IAAI,SAAS,CAAC;QAEtD,MAAM,EAAE,IAAI,EAAE,CACZ;YACE,aAAa,EAAE,GAAG;YAClB,MAAM,EAAE,YAAY;YACpB,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC;YACzB,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC;YACnC,YAAY,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YACjD,UAAU,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;YAC7C,aAAa,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC;YACnD,YAAY;SACb,EACD,uBAAuB,CACxB,CAAC;QAEF,OAAO;YACL,KAAK,EAAE,IAAa;YACpB,YAAY;YACZ,cAAc,EAAE,aAAa;YAC7B,QAAQ,EAAE,IAAI,IAAI,IAAI;YACtB,aAAa,EAAE,aAAa,CAAC,aAAa,IAAI,IAAI;YAClD,KAAK;YACL,YAAY;YACZ,eAAe;YACf,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,YAAY;YACpB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,SAAS,EAAE,aAAa,CAAC,SAAS;YAClC,MAAM,EAAE,aAAa,CAAC,MAAM;SAC7B,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,QAAQ,KAAK,YAAY,CAAC;IACxD,MAAM,IAAI,aAAa,CACrB,qBAAqB;QACnB,CAAC,CAAC,oBAAoB,CAAC,gBAAgB;QACvC,CAAC,CAAC,oBAAoB,CAAC,cAAc,EACvC,qBAAqB;QACnB,CAAC,CAAC,gJAAgJ,cAAc,CAAC,MAAM,uBAAuB;QAC9L,CAAC,CAAC,6BAA6B,SAAS,EAAE,EAC5C;QACE,YAAY,EAAE,QAAQ,IAAI,SAAS;QACnC,mBAAmB,EAAE,cAAc,CAAC,MAAM;KAC3C,CACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,SAAiB,EACjB,SAAiB;IAEjB,OAAO;QACL,GAAG,EAAE,EAAE,EAAE,wBAAwB;QACjC,UAAU,EAAE,SAAS,CAAC,MAAM;QAC5B,WAAW,EAAE,SAAS;QACtB,YAAY,EAAE,iBAAiB;QAC/B,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;QAC9B,aAAa,EAAE;YACb,cAAc,EAAE,IAAI;YACpB,kBAAkB,EAAE,gBAAgB;YACpC,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,QAAQ,EAAE,EAAE;SACb;KACF,CAAC;AACJ,CAAC"}

package/dist/normalization/pdf/prompt.d.ts

@@ -0,0 +1,3 @@
+export declare const PDF_EXTRACTION_SYSTEM: string;
+export declare const PDF_EXTRACTION_PROMPT: string;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/normalization/pdf/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/normalization/pdf/prompt.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,qBAAqB,QAOvB,CAAC;AAEZ,eAAO,MAAM,qBAAqB,QA0G0D,CAAC"}

package/dist/normalization/pdf/prompt.js

@@ -0,0 +1,118 @@
+import { SectionCategoryEnum } from "../normalized-doc-schema";
+const SECTION_CATEGORIES = SectionCategoryEnum.options.join(", ");
+export const PDF_EXTRACTION_SYSTEM = [
+    "You are a precise scientific paper parser.",
+    "You receive a PDF of a scientific paper and extract its full content into a structured JSON object.",
+    "Respond with strictly valid JSON only.",
+    "No markdown, no code fences, no comments, no extra text.",
+    "If a field is optional and you cannot determine its value, omit the key entirely.",
+    "Do not include trailing commas.",
+].join(" ");
+export const PDF_EXTRACTION_PROMPT = `Extract the full content of this scientific paper PDF into a JSON object with the following structure.
+
+## Top-level fields
+
+- "schemaVersion": always "v1"
+- "metadata": front-matter (see below)
+- "sections": array of section metadata (see below)
+- "blocks": array of content blocks (see below)
+- "citations": array of reference entries (see below)
+
+## metadata
+
+An object with these optional fields:
+- "ids": { "doi"?: string, "pmcid"?: string }
+- "title": string (the paper title)
+- "journal": { "title"?: string, "publisher"?: { "name"?: string } }
+- "pub_dates": [{ "pub_type"?: string, "year"?: string, "month"?: string, "day"?: string }]
+- "volume"?: string
+- "issue"?: string
+- "license": { "type"?: string, "href"?: string }
+- "abstract": { "paragraphs": [string, ...] } (split abstract into paragraphs)
+- "contributors": [{ "role"?: "author", "given"?: string, "surname"?: string, "full"?: string, "email"?: string, "xref_aff"?: ["aff1"] }]
+- "affiliations": [{ "id"?: "aff1", "label"?: "1", "text"?: string, "country"?: string }]
+- "keywords": [string, ...]
+
+## sections
+
+An array of objects, one per document section. Assign each section:
+- "id": a unique slug (e.g., "sec-introduction", "sec-methods")
+- "title": the section heading as written
+- "normalized_title": lowercase, trimmed heading
+- "category": one of: ${SECTION_CATEGORIES}
+- "level": integer depth (1 = top-level section, 2 = subsection, etc.)
+- "order": 0-based position among all sections
+- "anchor": same as id
+- "path_anchor": slash-separated ancestor path (e.g., "sec-methods/sec-cell-culture")
+
+## blocks
+
+An array of content blocks in document order. Each block is one of:
+
+### Paragraph block
+{
+  "kind": "paragraph",
+  "plain_text": "The full paragraph text...",
+  "section_id": "sec-introduction",
+  "path_anchor": "sec-introduction",
+  "order": 0
+}
+
+### Figure block
+{
+  "kind": "figure",
+  "id": "fig1",
+  "label": "Figure 1",
+  "title": "Figure title if any",
+  "plain_text": "Figure caption text",
+  "section_id": "sec-results",
+  "path_anchor": "sec-results",
+  "order": 5
+}
+
+### Table block
+{
+  "kind": "table",
+  "id": "tbl1",
+  "label": "Table 1",
+  "title": "Table title if any",
+  "plain_text": "Table caption text",
+  "section_id": "sec-results",
+  "path_anchor": "sec-results",
+  "order": 6
+}
+
+IMPORTANT for blocks:
+- Every block MUST have "section_id" and "path_anchor" fields (strings, required).
+- "order" should be a global 0-based index across all blocks.
+- Extract the FULL text of every paragraph. Do not summarize or truncate.
+
+## citations
+
+An array of reference list entries. For each reference:
+{
+  "id": "ref1",
+  "label": "1",
+  "text": "Full reference string as written",
+  "articleTitle"?: string,
+  "sourceTitle"?: string (journal name),
+  "year"?: number (integer),
+  "volume"?: string,
+  "issue"?: string,
+  "fpage"?: string,
+  "lpage"?: string,
+  "doi"?: string,
+  "pmid"?: string,
+  "authors"?: [{ "given"?: string, "surname"?: string }]
+}
+
+## Rules
+
+1. Extract ALL text content from the paper. Do not skip or summarize any section.
+2. Preserve paragraph boundaries as they appear in the paper.
+3. For section categories, use the most specific match from the allowed list. Use "other" only as a last resort.
+4. Number all blocks with a global "order" starting at 0.
+5. The "section_id" on each block must match an "id" in the sections array.
+6. Do not fabricate content. Only include what is actually in the document.
+7. For citations, extract as much structured metadata as you can parse from each reference.`;
+//# sourceMappingURL=prompt.js.map

package/dist/normalization/pdf/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../../src/normalization/pdf/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAE/D,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,4CAA4C;IAC5C,qGAAqG;IACrG,wCAAwC;IACxC,0DAA0D;IAC1D,mFAAmF;IACnF,iCAAiC;CAClC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBA+Bb,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4FA2EkD,CAAC"}

package/dist/sources/arxiv/client.d.ts

@@ -0,0 +1,4 @@
+export declare function buildArxivPdfUrl(arxivId: string): string;
+export declare function buildArxivAbsUrl(arxivId: string): string;
+export declare function buildAr5ivHtmlUrl(arxivId: string): string;
+//# sourceMappingURL=client.d.ts.map

package/dist/sources/arxiv/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/sources/arxiv/client.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAGxD;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAGxD;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAGzD"}

package/dist/sources/arxiv/client.js

@@ -0,0 +1,13 @@
+export function buildArxivPdfUrl(arxivId) {
+    const id = arxivId.replace(/^arxiv:/i, "");
+    return `https://arxiv.org/pdf/${id}.pdf`;
+}
+export function buildArxivAbsUrl(arxivId) {
+    const id = arxivId.replace(/^arxiv:/i, "");
+    return `https://arxiv.org/abs/${id}`;
+}
+export function buildAr5ivHtmlUrl(arxivId) {
+    const id = arxivId.replace(/^arxiv:/i, "");
+    return `https://ar5iv.org/html/${id}`;
+}
+//# sourceMappingURL=client.js.map

package/dist/sources/arxiv/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/sources/arxiv/client.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,OAAO,yBAAyB,EAAE,MAAM,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,OAAO,yBAAyB,EAAE,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,OAAO,0BAA0B,EAAE,EAAE,CAAC;AACxC,CAAC"}

package/dist/sources/biorxiv/client.d.ts

@@ -0,0 +1,21 @@
+import { type PreprintRecord } from "../preprint-discovery";
+export type SourceFetchOptions = {
+    UA?: string;
+    preferVersion?: number | null;
+};
+/** Return the versioned .source.xml record for the best-matching version (or null) */
+export declare function buildBiorxivLatestSourceXmlUrlViaApi(idOrDoi: string, opts?: SourceFetchOptions): Promise<PreprintRecord | null>;
+/** Fetch the JATS XML content via the API-discovered URL */
+export declare function fetchBiorxivSourceXmlViaApi(idOrDoi: string, opts?: SourceFetchOptions): Promise<{
+    url: string;
+    xml: string;
+    version?: number | null;
+} | null>;
+/** Build the canonical full HTML URL (versioned) from a biorxiv id/doi */
+export declare function buildBiorxivFullHtmlUrlFromId(idOrDoi: string): string | null;
+/** Fetch the full HTML via r.jina.ai proxy to bypass origin 403s */
+export declare function fetchBiorxivFullHtmlViaProxy(idOrDoiWithVersion: string, UA?: string): Promise<{
+    url: string;
+    html: string;
+} | null>;
+//# sourceMappingURL=client.d.ts.map

package/dist/sources/biorxiv/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/sources/biorxiv/client.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAI/B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAWF,sFAAsF;AACtF,wBAAsB,oCAAoC,CACxD,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,kBAAkB,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAMhC;AAED,4DAA4D;AAC5D,wBAAsB,2BAA2B,CAC/C,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,kBAAkB,GACxB,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAAC,CA6IvE;AAED,0EAA0E;AAC1E,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAQ5E;AAED,oEAAoE;AACpE,wBAAsB,4BAA4B,CAChD,kBAAkB,EAAE,MAAM,EAC1B,EAAE,SAA0B,GAC3B,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAkB/C"}