@webcrawl/mcp 1.4.2 → 1.4.4

package/dist/core/parse/pdf.js

@@ -0,0 +1,156 @@
+/**
+ * PDF to Markdown. Text pages are read locally with pdf-inspector (free, ~5 ms/page).
+ * Only pages it flags as scanned go to OCR. Anything not read is reported as skipped.
+ */
+import { createRequire } from "node:module";
+import { ocrAvailable, runOcr } from "./ocr.js";
+let inspector = null;
+function loadInspector() {
+    // Native addon: load with require so the bundler leaves it alone.
+    if (!inspector)
+        inspector = createRequire(import.meta.url)("@firecrawl/pdf-inspector");
+    return inspector;
+}
+const DEFAULT_MAX_OCR_PAGES = Number(process.env.WEBCRAWL_PARSE_MAX_OCR_PAGES) || 50;
+/**
+ * Decides which pages go where. Pure, so it can be unit-tested without a PDF.
+ */
+export function planPdfPages(input) {
+    const { pageCount, options } = input;
+    const mode = options.ocr ?? "auto";
+    const maxOcr = options.maxOcrPages ?? DEFAULT_MAX_OCR_PAGES;
+    const all = Array.from({ length: pageCount }, (_, i) => i + 1);
+    const selected = options.pages?.length
+        ? [...new Set(options.pages)].filter((p) => p >= 1 && p <= pageCount).sort((a, b) => a - b)
+        : all;
+    const flagged = new Set(input.needsOcr);
+    const native = [];
+    const wantOcr = [];
+    for (const p of selected) {
+        if (mode === "force" || (mode === "auto" && flagged.has(p)))
+            wantOcr.push(p);
+        else
+            native.push(p);
+    }
+    const skipped = [];
+    let ocr = [];
+    if (mode === "off") {
+        for (const p of selected)
+            if (flagged.has(p))
+                skipped.push({ page: p, reason: "ocr_disabled" });
+    }
+    else if (!input.ocrConfigured) {
+        for (const p of wantOcr)
+            skipped.push({ page: p, reason: "ocr_unavailable" });
+        // In force mode, still return whatever text the page has natively.
+        if (mode === "force")
+            native.push(...wantOcr.filter((p) => !flagged.has(p)));
+    }
+    else {
+        ocr = wantOcr.slice(0, maxOcr);
+        for (const p of wantOcr.slice(maxOcr))
+            skipped.push({ page: p, reason: "page_limit" });
+    }
+    native.sort((a, b) => a - b);
+    return { pageCount, native, ocr, skipped };
+}
+/** First real-looking h1/h2 (skipping arXiv/DOI stamps and page furniture), else first h3. */
+export function pickTitle(markdown) {
+    const clean = (s) => s.replace(/<\/?\w+>|[*_]/g, "").trim();
+    const usable = (s) => {
+        const t = clean(s);
+        return t.length >= 3 && t.length <= 160 && !/^(arxiv:|doi:|page \d|\d+$)/i.test(t);
+    };
+    const headings = [...markdown.matchAll(/^(#{1,3})\s+(.+)$/gm)].map((m) => ({ level: m[1].length, text: m[2] }));
+    const pick = headings.find((h) => h.level <= 2 && usable(h.text)) ?? headings.find((h) => usable(h.text));
+    return pick ? clean(pick.text) : "";
+}
+function toBuffer(bytes) {
+    return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+}
+/** Page count and pages needing OCR, without extracting text (cheap: ~1 ms/page). */
+export function pdfPageInfo(bytes) {
+    const info = loadInspector().detectPdf(toBuffer(bytes));
+    return { pageCount: info.pageCount, pagesNeedingOcr: info.pagesNeedingOcr, title: info.title?.trim() || undefined };
+}
+export async function parsePdf(bytes, options = {}) {
+    const started = Date.now();
+    const buf = toBuffer(bytes);
+    const pdf = loadInspector();
+    const warnings = [];
+    const info = pdf.detectPdf(buf);
+    const plan = planPdfPages({
+        pageCount: info.pageCount,
+        needsOcr: info.pagesNeedingOcr,
+        options,
+        ocrConfigured: ocrAvailable(),
+    });
+    if (info.hasEncodingIssues)
+        warnings.push("Some fonts have missing character maps; text may contain gaps.");
+    const byPage = new Map();
+    const nativeRead = [];
+    if (plan.native.length) {
+        // extractPagesMarkdown takes and returns 0-indexed pages; detectPdf reports 1-indexed ones.
+        const extracted = await pdf.extractPagesMarkdownAsync(buf, plan.native.map((p) => p - 1));
+        for (const raw of extracted.pages) {
+            const page = raw.page + 1;
+            const md = raw.markdown.trim();
+            if (md) {
+                byPage.set(page, md);
+                nativeRead.push(page);
+            }
+            else if (!plan.ocr.includes(page) && !plan.skipped.some((s) => s.page === page)) {
+                plan.skipped.push({ page, reason: "empty" });
+            }
+        }
+    }
+    const ocrRead = [];
+    let ocrProvider;
+    if (plan.ocr.length) {
+        try {
+            const result = await runOcr({ bytes: buf, mime: "application/pdf", pages: plan.ocr, signal: options.signal });
+            ocrProvider = result.provider;
+            warnings.push(...result.warnings.map((w) => `OCR fallback: ${w}`));
+            const wanted = new Set(plan.ocr);
+            for (const p of result.pages) {
+                if (!wanted.has(p.page) || !p.markdown)
+                    continue;
+                byPage.set(p.page, p.markdown);
+                ocrRead.push(p.page);
+            }
+            for (const p of plan.ocr)
+                if (!ocrRead.includes(p))
+                    plan.skipped.push({ page: p, reason: "ocr_failed" });
+        }
+        catch (err) {
+            warnings.push(`OCR failed: ${err instanceof Error ? err.message : String(err)}`);
+            for (const p of plan.ocr)
+                plan.skipped.push({ page: p, reason: "ocr_failed" });
+        }
+    }
+    const ordered = [...byPage.keys()].sort((a, b) => a - b);
+    const multi = ordered.length > 1;
+    const markdown = ordered
+        .map((p) => (multi ? `<!-- page ${p} -->\n\n${byPage.get(p)}` : byPage.get(p)))
+        .join("\n\n");
+    plan.skipped.sort((a, b) => a.page - b.page);
+    // Prefer document metadata, then the highest-level heading (first page banners are often ###).
+    const title = info.title?.trim() || pickTitle(markdown);
+    return {
+        title,
+        markdown,
+        meta: {
+            kind: "pdf",
+            mime: "application/pdf",
+            method: ocrRead.length > nativeRead.length ? "ocr" : "native",
+            bytes: buf.byteLength,
+            pages: info.pageCount,
+            pagesNative: nativeRead.sort((a, b) => a - b),
+            pagesOcr: ocrRead.sort((a, b) => a - b),
+            pagesSkipped: plan.skipped,
+            ocrProvider,
+            warnings,
+            processingMs: Date.now() - started,
+        },
+    };
+}

package/dist/core/parse/types.d.ts

@@ -0,0 +1,38 @@
+import type { FileKind } from "./detect.js";
+/** How the bulk of the content was produced. */
+export type ParseMethod = "native" | "ocr" | "vision" | "transcript";
+export interface SkippedPage {
+    page: number;
+    reason: "ocr_disabled" | "ocr_unavailable" | "ocr_failed" | "page_limit" | "empty";
+}
+/** Reported to the caller as `metadata.parse`, and used to charge credits. */
+export interface ParseMeta {
+    kind: FileKind;
+    mime: string;
+    method: ParseMethod;
+    bytes: number;
+    pages?: number;
+    pagesNative?: number[];
+    pagesOcr?: number[];
+    pagesSkipped?: SkippedPage[];
+    ocrProvider?: string;
+    durationSeconds?: number;
+    transcriptProvider?: string;
+    visionProvider?: string;
+    warnings: string[];
+    processingMs: number;
+}
+export interface ParsedFile {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+}
+export interface PdfParseOptions {
+    /** "auto": OCR only flagged pages; "force": OCR every page; "off": never OCR. */
+    ocr?: "auto" | "force" | "off";
+    /** 1-indexed page numbers to include. */
+    pages?: number[];
+    /** Hard cap on pages sent to OCR in one request. */
+    maxOcrPages?: number;
+    signal?: AbortSignal;
+}

package/dist/core/parse/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/parse/youtube.d.ts

@@ -0,0 +1,30 @@
+/**
+ * YouTube link -> timestamped transcript from the video's own captions.
+ * No audio download and no model call: we read the caption track YouTube already publishes.
+ * Videos without captions return an error rather than an empty or invented transcript.
+ */
+import type { Segment } from "./media.js";
+import type { ParsedFile } from "./types.js";
+export declare class YouTubeError extends Error {
+    readonly status: number;
+    constructor(message: string, status?: number);
+}
+/** Returns the 11-character video id for watch, short, embed, shorts, live and youtu.be links. */
+export declare function youtubeVideoId(input: string): string | undefined;
+/** Parses YouTube's timedtext XML (`<p t= d=>` in ms, optional `<s>` children). */
+export declare function parseTimedText(xml: string): Segment[];
+export declare function parseYouTube(videoId: string, options?: {
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<ParsedFile>;
+export interface YouTubeResult extends ParsedFile {
+    contentHash: string;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+}
+/** Same transcript for the same video and language, so successful reads are kept for 24 hours. Failures are never cached. */
+export declare function parseYouTubeCached(videoId: string, options?: {
+    language?: string;
+    signal?: AbortSignal;
+    bypassCache?: boolean;
+}): Promise<YouTubeResult>;

package/dist/core/parse/youtube.js

@@ -0,0 +1,264 @@
+import { transcriptMarkdown } from "./media.js";
+import { getCachedPageAsync, setCachedPage } from "../engine/cache.js";
+const PLAYER_URL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
+const MAX_ATTEMPTS = 3;
+const UA = "com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip";
+/** Tried in order; YouTube blocks some clients from datacenter IPs, so we fall through. */
+const CLIENTS = [
+    {
+        context: {
+            client: {
+                clientName: "ANDROID",
+                clientVersion: "20.10.38",
+                androidSdkVersion: 30,
+                hl: "en",
+            },
+        },
+        ua: UA,
+    },
+    {
+        context: {
+            client: {
+                clientName: "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
+                clientVersion: "2.0",
+                hl: "en",
+            },
+            thirdParty: { embedUrl: "https://www.youtube.com/" },
+        },
+        ua: "Mozilla/5.0 (SMART-TV; Linux; Tizen 6.5) AppleWebKit/537.36 (KHTML, like Gecko) 85.0.4183.93/6.5 TV Safari/537.36",
+    },
+    {
+        context: {
+            client: {
+                clientName: "WEB_EMBEDDED_PLAYER",
+                clientVersion: "1.20250310.01.00",
+                hl: "en",
+            },
+            thirdParty: { embedUrl: "https://www.google.com/" },
+        },
+        ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0 Safari/537.36",
+    },
+    {
+        context: {
+            client: {
+                clientName: "IOS",
+                clientVersion: "20.10.4",
+                deviceMake: "Apple",
+                deviceModel: "iPhone16,2",
+                osName: "iPhone",
+                osVersion: "18.3.2.22D82",
+                hl: "en",
+            },
+        },
+        ua: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)",
+    },
+];
+/**
+ * YouTube refuses most cloud-hosting IP ranges. When YOUTUBE_RELAY_URL and YOUTUBE_RELAY_TOKEN are set,
+ * requests go through a relay on a network YouTube accepts; otherwise they go straight out.
+ */
+function youtubeFetch(url, init) {
+    const relay = process.env.YOUTUBE_RELAY_URL;
+    const token = process.env.YOUTUBE_RELAY_TOKEN;
+    if (!relay || !token)
+        return fetch(url, init);
+    return fetch(relay, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+            url,
+            method: init.method,
+            headers: init.headers,
+            body: init.body,
+        }),
+        signal: init.signal,
+    });
+}
+export class YouTubeError extends Error {
+    status;
+    constructor(message, status = 422) {
+        super(message);
+        this.status = status;
+        this.name = "YouTubeError";
+    }
+}
+/** Returns the 11-character video id for watch, short, embed, shorts, live and youtu.be links. */
+export function youtubeVideoId(input) {
+    let u;
+    try {
+        u = new URL(input);
+    }
+    catch {
+        return undefined;
+    }
+    const host = u.hostname.replace(/^(www|m|music)\./, "");
+    const ok = (s) => s && /^[A-Za-z0-9_-]{11}$/.test(s) ? s : undefined;
+    if (host === "youtu.be")
+        return ok(u.pathname.slice(1).split("/")[0]);
+    if (host !== "youtube.com" && host !== "youtube-nocookie.com")
+        return undefined;
+    if (u.pathname === "/watch")
+        return ok(u.searchParams.get("v"));
+    const m = /^\/(?:embed|shorts|live|v)\/([^/?#]+)/.exec(u.pathname);
+    return m ? ok(m[1]) : undefined;
+}
+function decodeEntities(s) {
+    return s
+        .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
+        .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+        .replace(/"/g, '"')
+        .replace(/'/g, "'")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&amp;/g, "&");
+}
+/** Parses YouTube's timedtext XML (`<p t= d=>` in ms, optional `<s>` children). */
+export function parseTimedText(xml) {
+    const out = [];
+    for (const m of xml.matchAll(/<p\s([^>]*)>([\s\S]*?)<\/p>/g)) {
+        const t = /\bt="(\d+)"/.exec(m[1]);
+        if (!t)
+            continue;
+        const d = /\bd="(\d+)"/.exec(m[1]);
+        const text = decodeEntities(m[2].replace(/<[^>]+>/g, ""))
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!text)
+            continue;
+        const start = Number(t[1]) / 1000;
+        out.push({
+            start,
+            end: Math.round((start + (d ? Number(d[1]) / 1000 : 0)) * 1000) / 1000,
+            text,
+        });
+    }
+    return out;
+}
+function pickTrack(tracks, language) {
+    const want = (language || "en").toLowerCase().split("-")[0];
+    const byLang = tracks.filter((t) => t.languageCode.toLowerCase().split("-")[0] === want);
+    const pool = byLang.length ? byLang : tracks;
+    // Human-written captions beat auto-generated ones.
+    return pool.find((t) => t.kind !== "asr") ?? pool[0];
+}
+export async function parseYouTube(videoId, options = {}) {
+    const started = performance.now();
+    const signal = options.signal ?? AbortSignal.timeout(25_000);
+    let player;
+    let ua = UA;
+    let lastReason = "";
+    // The relay's addresses are shared, so YouTube occasionally flags one with a bot check.
+    // A retry usually leaves from a different address, so retry only that case.
+    for (let attempt = 0; attempt < MAX_ATTEMPTS && !player; attempt++) {
+        if (attempt > 0 && !/not a bot/i.test(lastReason))
+            break;
+        if (attempt > 0)
+            await new Promise((r) => setTimeout(r, 400 * attempt));
+        for (const c of CLIENTS) {
+            try {
+                const res = await youtubeFetch(PLAYER_URL, {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json", "User-Agent": c.ua },
+                    body: JSON.stringify({
+                        context: c.context,
+                        videoId,
+                        contentCheckOk: true,
+                        racyCheckOk: true,
+                    }),
+                    signal,
+                });
+                if (!res.ok) {
+                    lastReason = `HTTP ${res.status}`;
+                    continue;
+                }
+                const p = (await res.json());
+                if (p.playabilityStatus?.status === "OK") {
+                    player = p;
+                    ua = c.ua;
+                    break;
+                }
+                lastReason =
+                    p.playabilityStatus?.reason ||
+                        p.playabilityStatus?.status?.toLowerCase() ||
+                        "unavailable";
+            }
+            catch (err) {
+                lastReason = err instanceof Error ? err.message : String(err);
+            }
+        }
+    }
+    if (!player)
+        throw new YouTubeError(`This video can't be read (${lastReason})`);
+    const tracks = player.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+    if (!tracks.length)
+        throw new YouTubeError("This video has no captions, so there is no transcript to read");
+    const track = pickTrack(tracks, options.language);
+    const cap = await youtubeFetch(track.baseUrl, {
+        headers: { "User-Agent": ua },
+        signal,
+    });
+    if (!cap.ok)
+        throw new YouTubeError(`Could not load the captions (HTTP ${cap.status})`, 502);
+    const segments = parseTimedText(await cap.text());
+    if (!segments.length)
+        throw new YouTubeError("The captions for this video were empty");
+    const lengthSeconds = Number(player.videoDetails?.lengthSeconds);
+    const duration = Number.isFinite(lengthSeconds) && lengthSeconds > 0
+        ? lengthSeconds
+        : segments[segments.length - 1].end;
+    const warnings = [];
+    if (track.kind === "asr")
+        warnings.push("Captions are auto-generated by YouTube and may contain errors");
+    return {
+        title: player.videoDetails?.title || "",
+        markdown: `## Transcript\n\n${transcriptMarkdown(segments)}`,
+        meta: {
+            kind: "video",
+            mime: "video/youtube",
+            method: "transcript",
+            bytes: 0,
+            durationSeconds: duration,
+            transcriptProvider: "captions",
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}
+/** Same transcript for the same video and language, so successful reads are kept for 24 hours. Failures are never cached. */
+export async function parseYouTubeCached(videoId, options = {}) {
+    const key = `wc-parse://youtube:${videoId}/lang=${options.language ?? "default"}`;
+    if (!options.bypassCache) {
+        const hit = await getCachedPageAsync(key, "parse");
+        const meta = hit?.structuredData?.parse;
+        if (hit && meta) {
+            return {
+                title: hit.title,
+                markdown: hit.markdown,
+                meta,
+                contentHash: `youtube:${videoId}`,
+                cached: true,
+                cacheLayer: hit.cacheLayer || "postgres",
+            };
+        }
+    }
+    const parsed = await parseYouTube(videoId, options);
+    setCachedPage(key, "parse", {
+        title: parsed.title,
+        markdown: parsed.markdown,
+        links: [],
+        structuredData: {
+            parse: parsed.meta,
+        },
+        statusCode: 200,
+        contentHash: `youtube:${videoId}`,
+    });
+    return {
+        ...parsed,
+        contentHash: `youtube:${videoId}`,
+        cached: false,
+        cacheLayer: "miss",
+    };
+}

package/dist/core/ranker/decision.d.ts

@@ -0,0 +1,28 @@
+import type { DecisionModelId, SearchFilterContext, UserLocationContext } from "./types.js";
+/** What `decisionModel: "auto"` resolves to. Set WEBCRAWL_DECISION_MODEL to change it in prod. */
+export declare function defaultDecisionModel(): DecisionModelId;
+/**
+ * Stage 5: Vercel AI Gateway Decision Model Reranker (`baseten/mammoth-1` & `convaiinnovations/laya`).
+ * Scores all top-N candidates in ONE shared-prefix round-trip via `experimental_decide`.
+ */
+export declare function runDecisionModelReranker(query: string, candidates: Array<{
+    title: string;
+    url: string;
+    description: string;
+    highlights: string[];
+}>, modelOption: DecisionModelId, timeoutMs: number, userLocation?: UserLocationContext, filterContext?: SearchFilterContext): Promise<{
+    scores: Map<number, {
+        normalizedScore: number;
+        rawScore: number;
+        confidence: number;
+    }>;
+    modelUsed: string | null;
+} | null>;
+/**
+ * Full 5-Stage Document & Passage Ranking Pipeline:
+ * 1. Canonical URL Deduplication
+ * 2. Weighted Reciprocal Rank Fusion (W-RRF, k=60) + Multi-Engine Consensus
+ * 3. Fielded Okapi BM25F+ (title 3.5x, headings 2.2x, url 2.0x, description 1.5x, body 1.0x)
+ * 4. ColBERT-inspired Subword + Token MaxSim Passage Scoring & Highlight Extraction
+ * 5. Vercel AI Gateway Decision Model Reranking (`baseten/mammoth-1` / `convaiinnovations/laya`)
+ */

package/dist/core/ranker/decision.js

@@ -0,0 +1,151 @@
+import { createGateway } from "@ai-sdk/gateway";
+import { experimental_decide as decide } from "ai";
+import { resolveCountryDisplayName } from "./lexical.js";
+const DECISION_MODELS = [
+    "baseten/mammoth-1",
+    "baseten/elephant-1",
+    "convaiinnovations/laya",
+    "none",
+];
+/** What `decisionModel: "auto"` resolves to. Set WEBCRAWL_DECISION_MODEL to change it in prod. */
+export function defaultDecisionModel() {
+    const env = process.env.WEBCRAWL_DECISION_MODEL?.trim();
+    return env && DECISION_MODELS.includes(env) ? env : "baseten/mammoth-1";
+}
+/**
+ * Stage 5: Vercel AI Gateway Decision Model Reranker (`baseten/mammoth-1` & `convaiinnovations/laya`).
+ * Scores all top-N candidates in ONE shared-prefix round-trip via `experimental_decide`.
+ */
+export async function runDecisionModelReranker(query, candidates, modelOption, timeoutMs, userLocation, filterContext) {
+    // Operator switch: WEBCRAWL_DECISION_MODEL picks what "auto" means (or "none" for lexical-only),
+    // so a model price change can be absorbed without a deploy of new code.
+    if (modelOption === "auto")
+        modelOption = defaultDecisionModel();
+    if (modelOption === "none" || candidates.length === 0) {
+        return null;
+    }
+    const apiKey = process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+    if (!apiKey) {
+        return null;
+    }
+    // Primary model selection: `baseten/mammoth-1` (#1 on Jev Decision Index 71.24, ~350ms)
+    // or user-specified `convaiinnovations/laya` / `baseten/elephant-1`.
+    const primaryModel = modelOption === "auto" ? "baseten/mammoth-1" : modelOption;
+    const fallbackModels = primaryModel === "baseten/mammoth-1"
+        ? ["baseten/elephant-1", "convaiinnovations/laya"]
+        : ["baseten/mammoth-1", "baseten/elephant-1"];
+    const gw = createGateway({ apiKey });
+    const stateCandidates = candidates.map((c, idx) => ({
+        id: `doc_${idx}`,
+        title: c.title.slice(0, 180),
+        url: c.url.slice(0, 200),
+        snippet: (c.highlights[0] || c.description || "").slice(0, 360),
+    }));
+    const contextClauses = [];
+    if (filterContext?.category) {
+        contextClauses.push(`category="${filterContext.category}"`);
+    }
+    if (filterContext?.subcategory) {
+        const subStr = Array.isArray(filterContext.subcategory)
+            ? filterContext.subcategory.join(", ")
+            : filterContext.subcategory;
+        contextClauses.push(`subcategory="${subStr}"`);
+    }
+    if (userLocation?.city || userLocation?.region || userLocation?.country) {
+        const locParts = [
+            userLocation.city,
+            userLocation.region,
+            userLocation.countryName ||
+                resolveCountryDisplayName(userLocation.country),
+        ].filter(Boolean);
+        contextClauses.push(`userLocation="${locParts.join(", ")}"`);
+    }
+    if (filterContext?.customInstruction) {
+        contextClauses.push(`agentFilter="${filterContext.customInstruction}"`);
+    }
+    const contextSuffix = contextClauses.length > 0 ? ` (${contextClauses.join("; ")})` : "";
+    const questions = {};
+    for (let i = 0; i < stateCandidates.length; i++) {
+        questions[`doc_${i}`] = {
+            type: "score",
+            instructions: `Evaluate how relevant, authoritative, and aligned candidate doc_${i} is for the search query${contextSuffix}. Penalize entity homonym mismatches or wrong-region pages when userLocation or category is specified.`,
+            criteria: [
+                "Irrelevant, off-topic, wrong entity, wrong region, or spam",
+                "Tangentially related or generic syndication with weak match to the requested category/subcategory/location",
+                "Relevant and addresses a substantial part of the query and category/subcategory context",
+                "Highly relevant, authoritative, and directly satisfies the query, category, subcategory, and user location",
+            ],
+        };
+    }
+    try {
+        const decisionPromise = decide({
+            model: gw.decisionModel(primaryModel),
+            state: JSON.stringify({
+                query,
+                ...(filterContext?.category
+                    ? { category: filterContext.category }
+                    : {}),
+                ...(filterContext?.subcategory
+                    ? { subcategory: filterContext.subcategory }
+                    : {}),
+                ...(userLocation ? { userLocation } : {}),
+                candidates: stateCandidates,
+            }),
+            questions,
+            providerOptions: {
+                gateway: {
+                    models: fallbackModels,
+                },
+            },
+            abortSignal: AbortSignal.timeout(timeoutMs),
+        });
+        const result = await decisionPromise;
+        const scores = new Map();
+        for (let i = 0; i < stateCandidates.length; i++) {
+            const ans = result.answers?.[`doc_${i}`];
+            if (!ans) {
+                continue;
+            }
+            let rawScore = typeof ans.score === "number" ? ans.score : 0;
+            let maxProb = typeof ans.confidence === "number" ? ans.confidence : 0;
+            if (ans.probabilities) {
+                const probs = Array.isArray(ans.probabilities)
+                    ? ans.probabilities
+                    : [
+                        Number(ans.probabilities["0"] ?? 0),
+                        Number(ans.probabilities["1"] ?? 0),
+                        Number(ans.probabilities["2"] ?? 0),
+                        Number(ans.probabilities["3"] ?? 0),
+                    ];
+                const expected = 0 * (probs[0] || 0) +
+                    1 * (probs[1] || 0) +
+                    2 * (probs[2] || 0) +
+                    3 * (probs[3] || 0);
+                if (expected > 0 || rawScore === 0) {
+                    rawScore = expected;
+                }
+                maxProb = Math.max(maxProb, ...probs.filter((p) => Number.isFinite(p)));
+            }
+            const normalizedScore = Math.max(0, Math.min(1, rawScore / 3));
+            scores.set(i, {
+                normalizedScore,
+                rawScore: Number(rawScore.toFixed(4)),
+                confidence: Number((maxProb || 0.85).toFixed(4)),
+            });
+        }
+        const modelUsed = result.response?.modelId ||
+            primaryModel;
+        return { scores, modelUsed };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Full 5-Stage Document & Passage Ranking Pipeline:
+ * 1. Canonical URL Deduplication
+ * 2. Weighted Reciprocal Rank Fusion (W-RRF, k=60) + Multi-Engine Consensus
+ * 3. Fielded Okapi BM25F+ (title 3.5x, headings 2.2x, url 2.0x, description 1.5x, body 1.0x)
+ * 4. ColBERT-inspired Subword + Token MaxSim Passage Scoring & Highlight Extraction
+ * 5. Vercel AI Gateway Decision Model Reranking (`baseten/mammoth-1` / `convaiinnovations/laya`)
+ */

package/dist/core/ranker/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./lexical.js";
+export * from "./decision.js";
+import type { RawCandidateDocument, RerankOptions, RerankResult } from "./types.js";
+export declare function rankDocuments(query: string, rawCandidates: RawCandidateDocument[], options?: RerankOptions): Promise<RerankResult>;