@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/engine/stealth.js

@@ -0,0 +1,222 @@
+/**
+ * Stealth mode: read pages a bot check refuses to plain HTTP clients.
+ *
+ * One real Chromium session (the interact browser: Vercel Sandbox in prod,
+ * local Playwright in dev) opens the entry page, waits until the check clears,
+ * then reads every requested URL with that cleared session. Raw documents
+ * (robots.txt, sitemaps) are read with fetch() from inside the page so they
+ * carry the same cookies, IP and TLS fingerprint as the browser.
+ *
+ * Never fabricates content: a page still showing a check comes back `blocked`.
+ */
+import { isAntiBotChallenge } from "./fetcher.js";
+import { setDomainCookie, setDomainUserAgent } from "./fetcher.js";
+import { cookieHeaderFrom, getWarmSession, saveWarmSession, WARM_TTL_MS } from "./session.js";
+import { remoteStealthProvider } from "../interact/remote.js";
+/**
+ * True when a real browser can run here: our own (sandbox in prod, local
+ * Playwright in dev) or a remote provider. A remote browser still needs the
+ * sandbox or local Playwright to run the step script against it.
+ */
+export async function stealthAvailable() {
+    if (process.env.WEBCRAWL_STEALTH === "false")
+        return false;
+    const { interactBrowserAvailable } = await import("../interact/driver.js");
+    return interactBrowserAvailable();
+}
+/** True when this fetched document is a bot check, not the real content. */
+export function isBlockedResponse(status, body) {
+    return isAntiBotChallenge(body ?? "", status ?? 200);
+}
+/**
+ * Opens `entryUrl` in a real browser, waits for the bot check to clear, then
+ * reads `items` in order through that session. One browser per call.
+ */
+export async function stealthFetch(entryUrl, items, opts = {}) {
+    const t0 = Date.now();
+    if (!(await stealthAvailable())) {
+        return { solved: false, entry: null, docs: [], durationMs: 0, error: "Stealth browser not configured" };
+    }
+    const { startBrowser, runBrowserStep, stopBrowser } = await import("../interact/driver.js");
+    const provider = opts.remote === false ? null : remoteStealthProvider();
+    const domain = hostOf(entryUrl);
+    const proxyUrl = opts.proxy ?? null;
+    let proxyUser = "";
+    let proxyPass = "";
+    if (proxyUrl) {
+        try {
+            const parsed = new URL(proxyUrl);
+            proxyUser = decodeURIComponent(parsed.username);
+            proxyPass = decodeURIComponent(parsed.password);
+        }
+        catch {
+            /* bad proxy URL: browser launch will surface the error */
+        }
+    }
+    // A warm session for this domain means the challenge is already solved: short
+    // solve budget. A cold solve needs the full budget. Proxy runs never reuse
+    // warm sessions (their cookies were earned from a datacenter egress IP).
+    const hadWarm = provider || proxyUrl ? null : await getWarmSession(domain).catch(() => null);
+    const tag = provider ? { provider } : { warm: Boolean(hadWarm) };
+    let handle = null;
+    // Filled in by `finally` once the provider reports the session cost.
+    const out = {};
+    let capturedCookies = [];
+    let capturedUa = "";
+    try {
+        const budgetMs = opts.budgetMs ?? 90_000;
+        handle = await startBrowser({
+            timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000),
+            remote: provider,
+            domain: provider || proxyUrl ? undefined : domain,
+            proxy: proxyUrl ?? undefined,
+        });
+        const res = await runBrowserStep(handle, {
+            type: "fetchMany",
+            entry: entryUrl,
+            items: items.slice(0, 40),
+            budgetMs,
+            solveMs: hadWarm ? Math.min(opts.solveMs ?? 20_000, 8_000) : (opts.solveMs ?? 20_000),
+            followSitemaps: opts.followSitemaps ?? 0,
+            ...(proxyUser ? { proxyUser, proxyPass, proxyByteCap: 6_000_000 } : {}),
+        });
+        if (!res.ok) {
+            out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: res.error || "Stealth browser step failed", ...tag };
+        }
+        else {
+            const value = (res.data?.value ?? {});
+            capturedCookies = Array.isArray(value.cookies) ? value.cookies : [];
+            capturedUa = value.userAgent ?? "";
+            if (capturedUa)
+                setDomainUserAgent(domain, capturedUa);
+            if (capturedCookies.length > 0) {
+                setDomainCookie(domain, cookieHeaderFrom(capturedCookies));
+            }
+            out.result = {
+                solved: Boolean(value.solved),
+                entry: value.entry ?? null,
+                docs: Array.isArray(value.docs) ? value.docs : [],
+                durationMs: 0,
+                ...tag,
+            };
+        }
+    }
+    catch (err) {
+        out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: err instanceof Error ? err.message : String(err), ...tag };
+    }
+    finally {
+        const solved = out.result?.solved === true;
+        const keepWarm = !provider && solved && (opts.keepWarm ?? true) && Boolean(handle && handle.kind === "sandbox" && capturedUa);
+        const cost = handle
+            ? await stopBrowser(handle, {
+                keepWarm,
+                domain,
+                userAgent: capturedUa,
+                cookieHeader: capturedCookies.length > 0 ? cookieHeaderFrom(capturedCookies) : "",
+            }).catch(() => null)
+            : null;
+        if (out.result && cost)
+            out.result.providerCostUsd = cost.costUsd;
+        if (!keepWarm && solved && capturedCookies.length > 0) {
+            // Even without a warm sandbox, remember the solved cookies for replay on
+            // the lightweight tier (UA-paired; rejection there just escalates).
+            await saveWarmSession({ domain, sandboxId: "", userAgent: capturedUa, cookieHeader: cookieHeaderFrom(capturedCookies), expiresAt: Date.now() + WARM_TTL_MS })
+                .catch(() => { });
+        }
+        console.info("[stealth] session result", {
+            domain,
+            solved,
+            warm: Boolean(hadWarm),
+            keepWarm,
+            cookieCount: capturedCookies.length,
+            durationMs: Date.now() - t0,
+        });
+    }
+    const result = out.result;
+    result.durationMs = Date.now() - t0;
+    if (provider) {
+        console.info("[stealth] remote session", {
+            provider,
+            solved: result.solved,
+            docs: result.docs.length,
+            costUsd: result.providerCostUsd,
+            durationMs: result.durationMs,
+            error: result.error,
+        });
+    }
+    return result;
+}
+/** Hostname of a URL, or "" when unparseable. */
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return "";
+    }
+}
+/** Extracts `<loc>` entries from a sitemap or sitemap index. */
+export function parseSitemapLocs(xml) {
+    const pages = [];
+    const sitemaps = [];
+    const isIndex = /<sitemapindex[\s>]/i.test(xml);
+    for (const m of xml.matchAll(/<loc>\s*(?:<!\[CDATA\[)?([^<\]]+?)(?:\]\]>)?\s*<\/loc>/gi)) {
+        const loc = m[1]
+            .trim()
+            .replace(/&amp;/g, "&")
+            .replace(/&lt;/g, "<")
+            .replace(/&gt;/g, ">")
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;|&apos;/g, "'");
+        if (isIndex || /\.xml(\.gz)?(\?|$)/i.test(loc))
+            sitemaps.push(loc);
+        else
+            pages.push(loc);
+    }
+    return { pages, sitemaps };
+}
+/** Sitemap URLs declared in robots.txt. */
+export function sitemapsFromRobots(robots) {
+    return Array.from(robots.matchAll(/^\s*Sitemap:\s*(\S+)/gim)).map((m) => m[1].trim());
+}
+/**
+ * Finds a blocked site's pages through one stealth session: robots.txt and the
+ * usual sitemap paths, following robots-declared and child sitemaps in the same
+ * session. Also returns the rendered entry page so its links count when there
+ * is no sitemap.
+ */
+export async function stealthDiscover(entryUrl, opts = {}) {
+    const origin = new URL(entryUrl).origin;
+    const first = await stealthFetch(entryUrl, [
+        ...(opts.sitemapUrls ?? []).map((url) => ({ url, raw: true })),
+        { url: `${origin}/robots.txt`, raw: true },
+        { url: `${origin}/sitemap.xml`, raw: true },
+        { url: `${origin}/sitemap_index.xml`, raw: true },
+        { url: `${origin}/sitemap-index.xml`, raw: true },
+    ], { budgetMs: opts.budgetMs ?? 75_000, followSitemaps: 12, remote: opts.remote });
+    const { pages, sitemapsRead } = collectSitemapPages(first.docs);
+    return {
+        solved: first.solved,
+        sitemapPages: pages,
+        sitemapsRead,
+        entry: first.entry,
+        durationMs: first.durationMs,
+        ...(first.error ? { error: first.error } : {}),
+        ...(first.providerCostUsd !== undefined ? { providerCostUsd: first.providerCostUsd } : {}),
+    };
+}
+/** Page URLs from every readable sitemap among `docs` (robots.txt and blocked docs are skipped). */
+export function collectSitemapPages(docs) {
+    const pages = new Set();
+    const sitemapsRead = [];
+    for (const doc of docs) {
+        if (!doc.ok || !doc.body || doc.blocked || (doc.status ?? 200) >= 400)
+            continue;
+        if (/\/robots\.txt(\?|$)/i.test(doc.url) || !doc.body.includes("<loc>"))
+            continue;
+        sitemapsRead.push(doc.url);
+        for (const u of parseSitemapLocs(doc.body).pages)
+            pages.add(u);
+    }
+    return { pages: Array.from(pages), sitemapsRead };
+}

package/dist/core/engine.d.ts

@@ -0,0 +1 @@
+export * from "./engine/index.js";

package/dist/core/engine.js

@@ -0,0 +1 @@
+export * from "./engine/index.js";

package/dist/core/extract/business.d.ts

@@ -0,0 +1,28 @@
+export interface BusinessListing {
+    name: string;
+    category?: string;
+    description?: string;
+    address?: string;
+    city?: string;
+    state?: string;
+    postcode?: string;
+    country?: string;
+    phone?: string;
+    email?: string;
+    website?: string;
+    rating?: number;
+    reviewCount?: number;
+    priceRange?: string;
+    openingHours?: string | string[];
+    foundedYear?: number;
+    employeeCount?: string;
+    headquarters?: string;
+    latitude?: number;
+    longitude?: number;
+    url: string;
+    sourceDomain: string;
+    extractedVia: "json-state" | "rsc-payload" | "json-ld" | "scaledown-extract-1" | "serp-snippet";
+}
+export declare function extractBusinessListingsFromHtml(html: string, pageUrl: string, countryCode?: string): BusinessListing[];
+export declare function extractBusinessListingFromSerpResult(title: string, description: string, url: string, countryCode?: string): BusinessListing | null;
+export declare function dedupeBusinessListings(businesses: BusinessListing[], maxItems?: number): BusinessListing[];

package/dist/core/extract/business.js

@@ -0,0 +1,344 @@
+import { extractDomainHost } from "./currency.js";
+import { normalizeIdentity, resultDedupeKey } from "./dedupe-key.js";
+import { resolveAbsoluteUrl } from "./real-estate.js";
+const BUSINESS_SCHEMA_TYPES = new Set([
+    "localbusiness",
+    "organization",
+    "corporation",
+    "restaurant",
+    "foodestablishment",
+    "cafeorcoffeeshop",
+    "bakery",
+    "barorpub",
+    "store",
+    "autodealer",
+    "autorepair",
+    "medicalbusiness",
+    "dentist",
+    "physician",
+    "pharmacy",
+    "hospital",
+    "healthandbeautybusiness",
+    "dayspa",
+    "hairsalon",
+    "legalservice",
+    "attorney",
+    "accountingservice",
+    "financialservice",
+    "insuranceagency",
+    "realestateagent",
+    "homeandconstructionbusiness",
+    "plumber",
+    "electrician",
+    "hvacbusiness",
+    "generalcontractor",
+    "locksmith",
+    "movingcompany",
+    "professionalservice",
+    "employmentagency",
+    "travelagency",
+    "sportsactivitylocation",
+    "exerciseGym",
+]);
+function isBusinessSchemaType(rawType) {
+    const clean = rawType.toLowerCase().trim();
+    if (BUSINESS_SCHEMA_TYPES.has(clean)) {
+        return true;
+    }
+    return (clean.endsWith("business") ||
+        clean.endsWith("store") ||
+        clean.endsWith("service") ||
+        clean.endsWith("shop") ||
+        clean.endsWith("clinic") ||
+        clean.endsWith("agency") ||
+        clean.endsWith("contractor"));
+}
+function parseAddressObject(addr) {
+    if (!addr) {
+        return {};
+    }
+    if (typeof addr === "string") {
+        return { address: addr.trim() };
+    }
+    if (typeof addr === "object") {
+        const a = addr;
+        const street = typeof a.streetAddress === "string"
+            ? a.streetAddress.trim()
+            : typeof a.address1 === "string"
+                ? a.address1.trim()
+                : "";
+        const city = typeof a.addressLocality === "string"
+            ? a.addressLocality.trim()
+            : typeof a.city === "string"
+                ? a.city.trim()
+                : "";
+        const state = typeof a.addressRegion === "string"
+            ? a.addressRegion.trim()
+            : typeof a.state === "string"
+                ? a.state.trim()
+                : "";
+        const postcode = typeof a.postalCode === "string" || typeof a.postalCode === "number"
+            ? String(a.postalCode).trim()
+            : typeof a.zipCode === "string"
+                ? a.zipCode.trim()
+                : "";
+        const country = typeof a.addressCountry === "string"
+            ? a.addressCountry.trim()
+            : typeof a.addressCountry
+                ?.name === "string"
+                ? String(a.addressCountry.name).trim()
+                : undefined;
+        const full = [street, city, state, postcode].filter(Boolean).join(", ");
+        return {
+            address: full ||
+                (typeof a.formattedAddress === "string"
+                    ? a.formattedAddress
+                    : undefined),
+            city: city || undefined,
+            state: state || undefined,
+            postcode: postcode || undefined,
+            country,
+        };
+    }
+    return {};
+}
+function walkBusinessJsonNode(node, pageUrl, portal, countryCode, extractedVia, out, depth = 0) {
+    if (depth > 12 || out.length >= 45 || !node || typeof node !== "object") {
+        return;
+    }
+    if (Array.isArray(node)) {
+        for (const item of node) {
+            walkBusinessJsonNode(item, pageUrl, portal, countryCode, extractedVia, out, depth + 1);
+        }
+        return;
+    }
+    const obj = node;
+    const rawTypeVal = Array.isArray(obj["@type"])
+        ? String(obj["@type"][0] || "")
+        : String(obj["@type"] || obj.type || "");
+    const name = typeof obj.name === "string"
+        ? obj.name.trim()
+        : typeof obj.businessName === "string"
+            ? obj.businessName.trim()
+            : typeof obj.companyName === "string"
+                ? obj.companyName.trim()
+                : "";
+    const phone = typeof obj.telephone === "string"
+        ? obj.telephone.trim()
+        : typeof obj.phone === "string"
+            ? obj.phone.trim()
+            : typeof obj.displayPhone === "string"
+                ? obj.displayPhone.trim()
+                : undefined;
+    const email = typeof obj.email === "string"
+        ? obj.email.replace(/^mailto:/i, "").trim()
+        : undefined;
+    const addrInfo = parseAddressObject(obj.address || obj.location);
+    const aggRating = obj.aggregateRating;
+    const ratingRaw = aggRating?.ratingValue ?? obj.rating ?? obj.averageRating;
+    const ratingNum = typeof ratingRaw === "number" || typeof ratingRaw === "string"
+        ? Number.parseFloat(String(ratingRaw))
+        : undefined;
+    const reviewCountRaw = aggRating?.reviewCount ??
+        aggRating?.ratingCount ??
+        obj.reviewCount ??
+        obj.numberOfReviews;
+    const reviewCountNum = typeof reviewCountRaw === "number" || typeof reviewCountRaw === "string"
+        ? Number.parseInt(String(reviewCountRaw).replace(/,/g, ""), 10)
+        : undefined;
+    const isGenericOrgOnly = /^(organization|corporation|brand|newsmediaorganization)$/i.test(rawTypeVal);
+    const hasBusinessSignals = (isBusinessSchemaType(rawTypeVal) &&
+        (!isGenericOrgOnly ||
+            Boolean(addrInfo.address || phone || ratingNum !== undefined))) ||
+        Boolean(name.length >= 2 &&
+            name.length <= 120 &&
+            (phone ||
+                (addrInfo.address && (ratingNum !== undefined || obj.priceRange)) ||
+                (obj.businessId !== undefined && addrInfo.address)));
+    // Skip generic site-level WebSite / SearchAction / BreadcrumbList nodes
+    if (hasBusinessSignals &&
+        name &&
+        !/^(website|webpage|breadcrumblist|searchaction|itemlist|faqpage)$/i.test(rawTypeVal)) {
+        const geoObj = obj.geo;
+        const lat = typeof geoObj?.latitude === "number"
+            ? geoObj.latitude
+            : typeof obj.latitude === "number"
+                ? obj.latitude
+                : undefined;
+        const lng = typeof geoObj?.longitude === "number"
+            ? geoObj.longitude
+            : typeof obj.longitude === "number"
+                ? obj.longitude
+                : undefined;
+        let category = typeof obj.category === "string"
+            ? obj.category
+            : Array.isArray(obj.categories) && typeof obj.categories[0] === "string"
+                ? obj.categories[0]
+                : typeof obj.servesCuisine === "string"
+                    ? obj.servesCuisine
+                    : rawTypeVal && !/^(organization|localbusiness)$/i.test(rawTypeVal)
+                        ? rawTypeVal
+                        : undefined;
+        if (Array.isArray(obj.categories) &&
+            obj.categories[0] &&
+            typeof obj.categories[0] === "object") {
+            const firstCat = obj.categories[0];
+            if (typeof firstCat.title === "string") {
+                category = firstCat.title;
+            }
+            else if (typeof firstCat.name === "string") {
+                category = firstCat.name;
+            }
+        }
+        let openingHours;
+        if (typeof obj.openingHours === "string") {
+            openingHours = obj.openingHours;
+        }
+        else if (Array.isArray(obj.openingHours)) {
+            openingHours = obj.openingHours
+                .filter((h) => typeof h === "string")
+                .slice(0, 7);
+        }
+        const foundingDate = typeof obj.foundingDate === "string" ||
+            typeof obj.foundingDate === "number"
+            ? String(obj.foundingDate)
+            : undefined;
+        const foundedMatch = foundingDate?.match(/\b(18\d{2}|19\d{2}|20\d{2})\b/);
+        const itemUrl = typeof obj.url === "string"
+            ? resolveAbsoluteUrl(obj.url, pageUrl)
+            : pageUrl;
+        if (addrInfo.address ||
+            phone ||
+            email ||
+            ratingNum !== undefined ||
+            typeof obj.description === "string") {
+            out.push({
+                name,
+                category,
+                description: typeof obj.description === "string"
+                    ? obj.description.trim().slice(0, 280)
+                    : undefined,
+                address: addrInfo.address,
+                city: addrInfo.city,
+                state: addrInfo.state,
+                postcode: addrInfo.postcode,
+                country: addrInfo.country || countryCode,
+                phone,
+                email,
+                website: itemUrl,
+                rating: ratingNum && ratingNum > 0 && ratingNum <= 10
+                    ? Number(ratingNum.toFixed(1))
+                    : undefined,
+                reviewCount: reviewCountNum &&
+                    Number.isFinite(reviewCountNum) &&
+                    reviewCountNum > 0
+                    ? reviewCountNum
+                    : undefined,
+                priceRange: typeof obj.priceRange === "string"
+                    ? obj.priceRange.trim()
+                    : undefined,
+                openingHours,
+                foundedYear: foundedMatch
+                    ? Number.parseInt(foundedMatch[1], 10)
+                    : undefined,
+                latitude: lat,
+                longitude: lng,
+                url: itemUrl,
+                sourceDomain: extractDomainHost(itemUrl) || portal,
+                extractedVia,
+            });
+        }
+    }
+    for (const val of Object.values(obj)) {
+        if (val && typeof val === "object") {
+            walkBusinessJsonNode(val, pageUrl, portal, countryCode, extractedVia, out, depth + 1);
+        }
+    }
+}
+export function extractBusinessListingsFromHtml(html, pageUrl, countryCode) {
+    const out = [];
+    const portal = extractDomainHost(pageUrl) || "unknown";
+    // Tier 1: Schema.org JSON-LD
+    const jsonLdRegex = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+    let match;
+    while ((match = jsonLdRegex.exec(html)) !== null) {
+        try {
+            const parsed = JSON.parse(match[1].trim());
+            walkBusinessJsonNode(parsed, pageUrl, portal, countryCode, "json-ld", out);
+        }
+        catch {
+            // Ignore malformed JSON-LD
+        }
+    }
+    // Tier 2: Embedded JSON state (__NEXT_DATA__, application/json)
+    if (out.length < 3) {
+        const stateRegex = /<script[^>]+(?:id=["']__NEXT_DATA__["']|type=["']application\/json["'])[^>]*>([\s\S]*?)<\/script>/gi;
+        while ((match = stateRegex.exec(html)) !== null) {
+            try {
+                const raw = match[1].trim();
+                if (raw.length > 20 && raw.length < 2_000_000) {
+                    const parsed = JSON.parse(raw);
+                    walkBusinessJsonNode(parsed, pageUrl, portal, countryCode, "json-state", out);
+                }
+            }
+            catch {
+                // Ignore
+            }
+        }
+    }
+    return dedupeBusinessListings(out);
+}
+export function extractBusinessListingFromSerpResult(title, description, url, countryCode) {
+    const combined = `${title} ${description}`;
+    const cleanName = title.replace(/\s*[|\-–\u2014•:]\s*.*$/, "").trim();
+    if (!cleanName ||
+        cleanName.length < 2 ||
+        /\b(the\s+\d+\s+best|best\s+\d+|top\s+\d+|best\s+.*\s+in\b|places?\s+to\b|where\s+to\b|guide\s+to\b|directory\b|yellow\s+pages|yelp|tripadvisor|timeout|broadsheet|urban\s+list)\b/i.test(cleanName)) {
+        return null;
+    }
+    const phoneMatch = combined.match(/(?:\+?\d{1,3}[\s.-]?)?\(?\d{2,4}\)?[\s.-]\d{3,4}[\s.-]\d{3,4}\b/);
+    const ratingMatch = combined.match(/\b([1-5]\.\d)\s*(?:\/\s*5|stars?|★)/i) ||
+        combined.match(/\brating[:\s]+([1-5]\.\d)\b/i);
+    const reviewMatch = combined.match(/\b(\d[\d,]{0,6})\s+reviews?\b/i);
+    const priceRangeMatch = combined.match(/(?:^|\s)(\${1,4})(?:\s|[·•,]|$)/);
+    const addrMatch = description.match(/\b(\d{1,5}\s+[A-Z][a-zA-Z0-9\s.]{2,30}\s+(?:St|Street|Rd|Road|Ave|Avenue|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pl|Place|Pde|Parade|Hwy|Highway)(?:,\s*[A-Za-z\s]+)?)/);
+    // Require at least one concrete business attribute (address, phone, or rating) in the SERP snippet
+    if (!addrMatch && !phoneMatch && !ratingMatch) {
+        return null;
+    }
+    const sourceDomain = extractDomainHost(url) || "unknown";
+    return {
+        name: cleanName,
+        description: description.trim().slice(0, 240) || undefined,
+        address: addrMatch ? addrMatch[1].trim() : undefined,
+        country: countryCode,
+        phone: phoneMatch ? phoneMatch[0].trim() : undefined,
+        website: url,
+        rating: ratingMatch ? Number.parseFloat(ratingMatch[1]) : undefined,
+        reviewCount: reviewMatch
+            ? Number.parseInt(reviewMatch[1].replace(/,/g, ""), 10)
+            : undefined,
+        priceRange: priceRangeMatch ? priceRangeMatch[1] : undefined,
+        url,
+        sourceDomain,
+        extractedVia: "serp-snippet",
+    };
+}
+export function dedupeBusinessListings(businesses, maxItems = 30) {
+    const seen = new Set();
+    const out = [];
+    for (const b of businesses) {
+        const normName = normalizeIdentity(b.name);
+        // Same name on different directory pages / URLs is kept; only exact repeats dropped.
+        const key = resultDedupeKey(b.url || b.website, normName);
+        if (!normName || normName.length < 2 || seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        out.push(b);
+        if (out.length >= maxItems) {
+            break;
+        }
+    }
+    return out;
+}

package/dist/core/extract/currency.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Derive ISO 4217 currency code from a raw price string and/or 2-letter ISO country code.
+ * Uses standard currency symbols and ISO country conventions without hardcoded retailer lists.
+ */
+export declare function resolveCurrencyCode(rawPriceOrCurrency?: string, countryCode?: string): string;
+/**
+ * Parse a numeric price and currency from a human price string such as
+ * "$360.41", "A$399.00", "£173.49", "1.299,00 €", "₹1,19,900", "₹1.45 Crores", "S$429", "7.490.000 ₫".
+ */
+export declare function parsePriceString(raw: string | number | undefined, countryCode?: string, explicitCurrency?: string, maxAllowedPrice?: number): {
+    price: number;
+    rawPrice: string;
+    currency: string;
+} | null;
+export declare function extractDomainHost(url: string): string;