@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/extract/cve.js

@@ -0,0 +1,397 @@
+// Structured CVE records for the `security` search category.
+// Sources: NVD CVE API 2.0 (scores, CWE, CPE products, KEV flag) and OSV
+// (ecosystem package ranges and fixed versions, GHSA <-> CVE aliases).
+const CVE_RE = /\bCVE-(\d{4})-(\d{4,7})\b/gi;
+const GHSA_RE = /\bGHSA(?:-[23456789cfghjmpqrvwx]{4}){3}\b/gi;
+export const SEVERITY_ORDER = {
+    critical: 4,
+    high: 3,
+    medium: 2,
+    low: 1,
+    none: 0,
+};
+const roundUp1 = (x) => {
+    const i = Math.round(x * 100000);
+    return i % 10000 === 0 ? i / 100000 : (Math.floor(i / 10000) + 1) / 10;
+};
+/** CVSS v3.0/v3.1 base score from a vector string (FIRST spec). Undefined for v4 or malformed vectors. */
+export function cvss3BaseScore(vector) {
+    if (!/^CVSS:3\.[01]\//.test(vector))
+        return undefined;
+    const m = Object.fromEntries(vector.split("/").slice(1).map((p) => p.split(":")));
+    const AV = { N: 0.85, A: 0.62, L: 0.55, P: 0.2 }[m.AV];
+    const AC = { L: 0.77, H: 0.44 }[m.AC];
+    const UI = { N: 0.85, R: 0.62 }[m.UI];
+    const changed = m.S === "C";
+    const PR = (changed ? { N: 0.85, L: 0.68, H: 0.5 } : { N: 0.85, L: 0.62, H: 0.27 })[m.PR];
+    const cia = { H: 0.56, L: 0.22, N: 0 };
+    const C = cia[m.C];
+    const I = cia[m.I];
+    const A = cia[m.A];
+    if ([AV, AC, UI, PR, C, I, A].some((v) => v === undefined) || (m.S !== "U" && m.S !== "C"))
+        return undefined;
+    const iss = 1 - (1 - C) * (1 - I) * (1 - A);
+    const impact = changed ? 7.52 * (iss - 0.029) - 3.25 * Math.pow(iss - 0.02, 15) : 6.42 * iss;
+    if (impact <= 0)
+        return 0;
+    const exploitability = 8.22 * AV * AC * PR * UI;
+    return changed
+        ? roundUp1(Math.min(1.08 * (impact + exploitability), 10))
+        : roundUp1(Math.min(impact + exploitability, 10));
+}
+/** CVSS v3/v4 qualitative rating bands. */
+export function severityFromScore(score) {
+    if (score === undefined || !Number.isFinite(score))
+        return undefined;
+    if (score >= 9)
+        return "critical";
+    if (score >= 7)
+        return "high";
+    if (score >= 4)
+        return "medium";
+    if (score > 0)
+        return "low";
+    return "none";
+}
+function normalizeSeverity(raw) {
+    const s = String(raw ?? "").trim().toLowerCase();
+    if (s === "critical")
+        return "critical";
+    if (s === "high" || s === "important")
+        return "high";
+    if (s === "medium" || s === "moderate")
+        return "medium";
+    if (s === "low")
+        return "low";
+    if (s === "none")
+        return "none";
+    return undefined;
+}
+export function normalizeCveId(id) {
+    return id.trim().toUpperCase();
+}
+/** Unique CVE ids in order of first appearance. */
+export function extractCveIdsFromText(text, max = 20) {
+    const out = [];
+    const seen = new Set();
+    for (const m of text.matchAll(CVE_RE)) {
+        const id = normalizeCveId(m[0]);
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        out.push(id);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+export function extractGhsaIdsFromText(text, max = 10) {
+    const out = new Set();
+    for (const m of text.matchAll(GHSA_RE)) {
+        out.add(`GHSA${m[0].slice(4).toLowerCase()}`);
+        if (out.size >= max)
+            break;
+    }
+    return [...out];
+}
+export function looksLikeVulnId(query) {
+    return /\bCVE-\d{4}-\d{4,7}\b/i.test(query) || /\bGHSA(?:-[a-z0-9]{4}){3}\b/i.test(query);
+}
+export const nvdUrl = (id) => `https://nvd.nist.gov/vuln/detail/${id}`;
+function englishDescription(list) {
+    if (!Array.isArray(list))
+        return "";
+    const en = list.find((d) => d?.lang === "en") ?? list[0];
+    return String(en?.value ?? "").replace(/\s+/g, " ").trim();
+}
+function pickNvdCvss(metrics) {
+    if (!metrics)
+        return {};
+    const order = [
+        ["cvssMetricV40", "4.0"],
+        ["cvssMetricV31", "3.1"],
+        ["cvssMetricV30", "3.0"],
+        ["cvssMetricV2", "2.0"],
+    ];
+    for (const [key, version] of order) {
+        const list = metrics[key];
+        if (!Array.isArray(list) || list.length === 0)
+            continue;
+        // Prefer NVD's own ("Primary") assessment over a CNA's secondary one.
+        const m = list.find((x) => x?.type === "Primary") ?? list[0];
+        const data = m?.cvssData ?? {};
+        const score = typeof data.baseScore === "number" ? data.baseScore : undefined;
+        return {
+            cvssScore: score,
+            cvssVector: data.vectorString,
+            cvssVersion: version,
+            severity: normalizeSeverity(data.baseSeverity ?? m?.baseSeverity) ?? severityFromScore(score),
+        };
+    }
+    return {};
+}
+function rangeText(m) {
+    const parts = [];
+    if (m.versionStartIncluding)
+        parts.push(`>=${m.versionStartIncluding}`);
+    if (m.versionStartExcluding)
+        parts.push(`>${m.versionStartExcluding}`);
+    if (m.versionEndIncluding)
+        parts.push(`<=${m.versionEndIncluding}`);
+    if (m.versionEndExcluding)
+        parts.push(`<${m.versionEndExcluding}`);
+    return parts.length ? parts.join(" ") : undefined;
+}
+function nvdAffected(configurations) {
+    if (!Array.isArray(configurations))
+        return [];
+    const byKey = new Map();
+    const walk = (nodes) => {
+        if (!Array.isArray(nodes))
+            return;
+        for (const node of nodes) {
+            for (const m of (node?.cpeMatch ?? [])) {
+                if (!m?.vulnerable || typeof m.criteria !== "string")
+                    continue;
+                // cpe:2.3:<part>:<vendor>:<product>:<version>:...
+                const [, , , vendor, product, version] = m.criteria.split(":");
+                if (!product || product === "*")
+                    continue;
+                const key = `${vendor}:${product}`;
+                const entry = byKey.get(key) ??
+                    { vendor: vendor && vendor !== "*" ? vendor : undefined, product, _versions: [] };
+                const range = rangeText(m) ?? (version && version !== "*" && version !== "-" ? version : undefined);
+                if (range && !entry._versions.includes(range))
+                    entry._versions.push(range);
+                if (m.versionEndExcluding && !entry.fixed)
+                    entry.fixed = m.versionEndExcluding;
+                byKey.set(key, entry);
+            }
+            walk(node?.children);
+        }
+    };
+    for (const c of configurations)
+        walk(c?.nodes);
+    return [...byKey.values()].slice(0, 8).map(({ _versions, ...a }) => ({
+        ...a,
+        ...(_versions.length ? { versions: _versions.slice(0, 4).join(", ") } : {}),
+    }));
+}
+/** One NVD `vulnerabilities[].cve` object (or the wrapper) -> CveRecord. */
+export function parseNvdCve(input) {
+    const cve = input?.cve ?? input;
+    if (!cve || typeof cve.id !== "string")
+        return null;
+    const id = normalizeCveId(cve.id);
+    if (!/^CVE-\d{4}-\d{4,}$/.test(id))
+        return null;
+    const summary = englishDescription(cve.descriptions);
+    const cwe = new Set();
+    for (const w of (cve.weaknesses ?? [])) {
+        for (const d of (w?.description ?? [])) {
+            const v = String(d?.value ?? "");
+            if (/^CWE-\d+$/.test(v))
+                cwe.add(v);
+        }
+    }
+    const affected = nvdAffected(cve.configurations);
+    const refs = (cve.references ?? [])
+        .map((r) => r?.url)
+        .filter((u) => typeof u === "string" && /^https?:\/\//.test(u));
+    return {
+        id,
+        ...(cve.cisaVulnerabilityName ? { title: String(cve.cisaVulnerabilityName) } : {}),
+        summary,
+        ...pickNvdCvss(cve.metrics),
+        ...(cwe.size ? { cwe: [...cwe] } : {}),
+        ...(affected.length ? { affected } : {}),
+        ...(cve.published ? { published: toIso(cve.published) } : {}),
+        ...(cve.lastModified ? { modified: toIso(cve.lastModified) } : {}),
+        ...(cve.cisaExploitAdd ? { kev: true } : {}),
+        references: [...new Set(refs)].slice(0, 12),
+        url: nvdUrl(id),
+        source: "nvd",
+    };
+}
+/** NVD 2.0 response body -> records. */
+export function parseNvdResponse(body) {
+    const list = Array.isArray(body?.vulnerabilities) ? body.vulnerabilities : [];
+    return list.map(parseNvdCve).filter((r) => r !== null);
+}
+function toIso(s) {
+    // NVD omits the zone ("2024-03-29T17:15:21.150"); treat as UTC.
+    const withZone = /[zZ]|[+-]\d\d:?\d\d$/.test(s) ? s : `${s}Z`;
+    const d = new Date(withZone);
+    return Number.isNaN(d.getTime()) ? s : d.toISOString();
+}
+function osvAffected(affected) {
+    if (!Array.isArray(affected))
+        return [];
+    const out = [];
+    for (const a of affected) {
+        const name = a?.package?.name;
+        if (typeof name !== "string" || !name)
+            continue;
+        const ranges = [];
+        let fixed;
+        for (const r of (a.ranges ?? [])) {
+            if (r?.type === "GIT")
+                continue;
+            let intro;
+            for (const e of (r?.events ?? [])) {
+                if (e.introduced !== undefined)
+                    intro = String(e.introduced);
+                if (e.fixed !== undefined) {
+                    fixed ??= String(e.fixed);
+                    ranges.push(intro && intro !== "0" ? `>=${intro} <${e.fixed}` : `<${e.fixed}`);
+                    intro = undefined;
+                }
+                if (e.last_affected !== undefined) {
+                    ranges.push(intro && intro !== "0" ? `>=${intro} <=${e.last_affected}` : `<=${e.last_affected}`);
+                    intro = undefined;
+                }
+            }
+            if (intro !== undefined)
+                ranges.push(intro === "0" ? "all versions" : `>=${intro}`);
+        }
+        if (!ranges.length && Array.isArray(a.versions) && a.versions.length) {
+            ranges.push(a.versions.slice(0, 4).join(", ") + (a.versions.length > 4 ? ", …" : ""));
+        }
+        out.push({
+            product: name,
+            ...(a.package.ecosystem ? { ecosystem: String(a.package.ecosystem) } : {}),
+            ...(ranges.length ? { versions: [...new Set(ranges)].slice(0, 4).join(", ") } : {}),
+            ...(fixed ? { fixed } : {}),
+        });
+    }
+    return out.slice(0, 8);
+}
+/** One OSV vulnerability object -> CveRecord, keyed by its CVE alias when it has one. */
+export function parseOsvVuln(v) {
+    if (typeof v?.id !== "string")
+        return null;
+    const ids = [v.id, ...(Array.isArray(v.aliases) ? v.aliases : [])].map(String);
+    const cveId = ids.find((x) => /^CVE-\d{4}-\d{4,}$/i.test(x));
+    const id = cveId ? normalizeCveId(cveId) : v.id;
+    const aliases = [...new Set(ids.filter((x) => normalizeCveId(x) !== normalizeCveId(id)))];
+    const vector = (v.severity ?? []).find((s) => /^CVSS_V[34]/.test(s?.type))?.score;
+    const sevRaw = v.database_specific?.severity ?? v.affected?.[0]?.database_specific?.severity;
+    const cweIds = v.database_specific?.cwe_ids;
+    const affected = osvAffected(v.affected);
+    const refs = (v.references ?? [])
+        .map((r) => r?.url)
+        .filter((u) => typeof u === "string" && /^https?:\/\//.test(u));
+    const summary = String(v.summary || v.details || "").replace(/\s+/g, " ").trim();
+    const score = typeof vector === "string" ? cvss3BaseScore(vector) : undefined;
+    return {
+        id,
+        ...(aliases.length ? { aliases } : {}),
+        ...(v.summary && v.details ? { title: String(v.summary) } : {}),
+        summary: summary.length > 600 ? `${summary.slice(0, 597)}…` : summary,
+        ...(score !== undefined ? { cvssScore: score } : {}),
+        ...(normalizeSeverity(sevRaw) || severityFromScore(score)
+            ? { severity: normalizeSeverity(sevRaw) ?? severityFromScore(score) }
+            : {}),
+        ...(typeof vector === "string"
+            ? { cvssVector: vector, cvssVersion: vector.startsWith("CVSS:4.0") ? "4.0" : vector.startsWith("CVSS:3.0") ? "3.0" : "3.1" }
+            : {}),
+        ...(Array.isArray(cweIds) && cweIds.length ? { cwe: cweIds.map(String) } : {}),
+        ...(affected.length ? { affected } : {}),
+        ...(v.published ? { published: toIso(v.published) } : {}),
+        ...(v.modified ? { modified: toIso(v.modified) } : {}),
+        references: [...new Set(refs)].slice(0, 12),
+        url: cveId ? nvdUrl(id) : `https://osv.dev/vulnerability/${v.id}`,
+        source: "osv",
+    };
+}
+/**
+ * Merge records for the same vulnerability across sources (matched on id or any alias).
+ * NVD wins for scores, CWE and KEV; OSV fills ecosystem packages and fixed versions.
+ */
+export function mergeCveRecords(records, max = 25) {
+    const byKey = new Map();
+    const keyOf = new Map();
+    const rank = (r) => (r.source === "nvd" ? 0 : r.source === "osv" ? 1 : 2);
+    for (const rec of [...records].sort((a, b) => rank(a) - rank(b))) {
+        const names = [rec.id, ...(rec.aliases ?? [])].map(normalizeCveId);
+        const existingKey = names.map((n) => keyOf.get(n)).find(Boolean);
+        if (!existingKey) {
+            byKey.set(rec.id, { ...rec });
+            for (const n of names)
+                keyOf.set(n, rec.id);
+            continue;
+        }
+        const cur = byKey.get(existingKey);
+        const aliasMap = new Map();
+        for (const a of [...(cur.aliases ?? []), rec.id, ...(rec.aliases ?? [])]) {
+            const n = normalizeCveId(a);
+            if (n !== normalizeCveId(cur.id) && !aliasMap.has(n))
+                aliasMap.set(n, a);
+        }
+        const aliases = new Set(aliasMap.values());
+        const affected = [...(cur.affected ?? [])];
+        for (const a of rec.affected ?? []) {
+            const match = affected.find((x) => x.product.toLowerCase() === a.product.toLowerCase());
+            if (!match)
+                affected.push(a);
+            else {
+                match.ecosystem ??= a.ecosystem;
+                match.fixed ??= a.fixed;
+                match.versions ??= a.versions;
+            }
+        }
+        byKey.set(existingKey, {
+            ...cur,
+            aliases: aliases.size ? [...aliases] : undefined,
+            title: cur.title ?? rec.title,
+            summary: cur.summary || rec.summary,
+            severity: cur.severity ?? rec.severity,
+            cvssScore: cur.cvssScore ?? rec.cvssScore,
+            cvssVector: cur.cvssVector ?? rec.cvssVector,
+            cvssVersion: cur.cvssVersion ?? rec.cvssVersion,
+            cwe: cur.cwe ?? rec.cwe,
+            affected: affected.length ? affected.slice(0, 8) : undefined,
+            published: cur.published ?? rec.published,
+            modified: cur.modified ?? rec.modified,
+            kev: cur.kev || rec.kev || undefined,
+            references: [...new Set([...cur.references, ...rec.references])].slice(0, 16),
+        });
+        for (const n of names)
+            keyOf.set(n, existingKey);
+    }
+    const out = [...byKey.values()].map((r) => {
+        const clean = Object.fromEntries(Object.entries(r).filter(([, v]) => v !== undefined));
+        if (!clean.severity && clean.cvssScore !== undefined)
+            clean.severity = severityFromScore(clean.cvssScore);
+        return clean;
+    });
+    return sortCveRecords(out).slice(0, max);
+}
+/** Exact-id hits first, then KEV, then severity, then newest. */
+export function sortCveRecords(records, pinnedIds = []) {
+    const pinned = new Set(pinnedIds.map(normalizeCveId));
+    const isPinned = (r) => pinned.has(r.id) || (r.aliases ?? []).some((a) => pinned.has(normalizeCveId(a)));
+    return [...records].sort((a, b) => {
+        const p = Number(isPinned(b)) - Number(isPinned(a));
+        if (p)
+            return p;
+        const k = Number(!!b.kev) - Number(!!a.kev);
+        if (k)
+            return k;
+        const s = SEVERITY_ORDER[b.severity ?? "none"] - SEVERITY_ORDER[a.severity ?? "none"];
+        if (s)
+            return s;
+        const sc = (b.cvssScore ?? -1) - (a.cvssScore ?? -1);
+        if (sc)
+            return sc;
+        return (b.published ?? "").localeCompare(a.published ?? "");
+    });
+}
+/** "9.8 Critical" / "High" for cards and MCP text. */
+export function cveSeverityLabel(r) {
+    const sev = r.severity ?? severityFromScore(r.cvssScore);
+    const word = sev ? sev[0].toUpperCase() + sev.slice(1) : undefined;
+    if (r.cvssScore !== undefined && word)
+        return `${r.cvssScore.toFixed(1)} ${word}`;
+    return word;
+}

package/dist/core/extract/dedupe-key.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Canonical form of a result URL for duplicate detection: lower-case host without
+ * `www.`, no hash, no tracking params, no trailing slash. Distinct paths/queries stay
+ * distinct, so two listings on the same provider are never merged.
+ */
+export declare function canonicalResultUrl(rawUrl: string | undefined): string;
+export declare function normalizeIdentity(text: string | undefined): string;
+/**
+ * Duplicate key = same URL AND same identity (address / name). Different URLs from the
+ * same provider are kept, and so is one search-results URL that yields several items.
+ */
+export declare function resultDedupeKey(url: string | undefined, identity: string | undefined): string;

package/dist/core/extract/dedupe-key.js

@@ -0,0 +1,36 @@
+const TRACKING_PARAM = /^(utm_|fbclid$|gclid$|msclkid$|mc_cid$|mc_eid$|_ga$|_gl$|ref$|ref_src$)/i;
+/**
+ * Canonical form of a result URL for duplicate detection: lower-case host without
+ * `www.`, no hash, no tracking params, no trailing slash. Distinct paths/queries stay
+ * distinct, so two listings on the same provider are never merged.
+ */
+export function canonicalResultUrl(rawUrl) {
+    if (!rawUrl) {
+        return "";
+    }
+    try {
+        const u = new URL(rawUrl.trim());
+        u.hash = "";
+        u.hostname = u.hostname.toLowerCase().replace(/^www\./, "");
+        for (const key of Array.from(u.searchParams.keys())) {
+            if (TRACKING_PARAM.test(key)) {
+                u.searchParams.delete(key);
+            }
+        }
+        u.searchParams.sort();
+        return `${u.hostname}${u.pathname.replace(/\/+$/, "")}${u.search}`;
+    }
+    catch {
+        return rawUrl.trim().toLowerCase();
+    }
+}
+export function normalizeIdentity(text) {
+    return (text ?? "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+}
+/**
+ * Duplicate key = same URL AND same identity (address / name). Different URLs from the
+ * same provider are kept, and so is one search-results URL that yields several items.
+ */
+export function resultDedupeKey(url, identity) {
+    return `${canonicalResultUrl(url)}::${normalizeIdentity(identity)}`;
+}

package/dist/core/extract/flights.d.ts

@@ -0,0 +1,37 @@
+export interface FlightOffer {
+    title: string;
+    airline?: string;
+    flightNumber?: string;
+    origin?: string;
+    destination?: string;
+    departureTime?: string;
+    arrivalTime?: string;
+    duration?: string;
+    stops?: number | string;
+    cabinClass?: "economy" | "premium_economy" | "business" | "first";
+    tripType?: "one_way" | "round_trip";
+    price?: number;
+    priceText: string;
+    currency?: string;
+    url: string;
+    provider: string;
+    extractedVia: "json-state" | "rsc-payload" | "json-ld" | "scaledown-extract-1" | "serp-snippet";
+}
+export declare function inferRouteFromText(text: string): {
+    origin?: string;
+    destination?: string;
+};
+export declare function inferCabinClass(text: string): "economy" | "premium_economy" | "business" | "first" | undefined;
+export declare function inferTripType(text: string): "one_way" | "round_trip" | undefined;
+export declare function inferDurationAndStops(text: string): {
+    duration?: string;
+    stops?: number | string;
+};
+export declare function parseFlightPrice(rawPrice: string | number | undefined | null, countryCode?: string, explicitCurrency?: string): {
+    price?: number;
+    priceText: string;
+    currency: string;
+};
+export declare function extractFlightOffersFromHtml(html: string, pageUrl: string, countryCode?: string, queryHint?: string): FlightOffer[];
+export declare function extractFlightOfferFromSerpResult(title: string, description: string, url: string, countryCode?: string, queryHint?: string): FlightOffer | null;
+export declare function dedupeFlightOffers(offers: FlightOffer[], maxItems?: number): FlightOffer[];