indiecrm-cli 0.1.0 → 0.2.1

package/dist/linkedin-products.js

@@ -0,0 +1,786 @@
+import { load } from "cheerio";
+import { DEFAULT_LINKEDIN_SCRAPER_USER_AGENT } from "./linkedin-session-contracts.js";
+const DEFAULT_LINKEDIN_BASE_URL = "https://www.linkedin.com";
+function normalizeWhitespace(value) {
+    return (value ?? "").replace(/\s+/g, " ").trim();
+}
+function normalizeDomainInput(value) {
+    return value
+        .trim()
+        .toLowerCase()
+        .replace(/^https?:\/\//, "")
+        .replace(/^www\./, "")
+        .split("/")[0] ?? "";
+}
+function getLinkedInBaseUrl(env = process.env) {
+    const override = env.SALESPROMPTER_LINKEDIN_BASE_URL?.trim();
+    if (!override) {
+        return DEFAULT_LINKEDIN_BASE_URL;
+    }
+    return override.replace(/\/+$/, "");
+}
+function isAllowedLinkedInHostname(hostname, env = process.env) {
+    if (/(^|\.)linkedin\.com$/i.test(hostname)) {
+        return true;
+    }
+    try {
+        return hostname.toLowerCase() === new URL(getLinkedInBaseUrl(env)).hostname.toLowerCase();
+    }
+    catch {
+        return false;
+    }
+}
+function buildLinkedInUrl(pathname, env = process.env) {
+    return new URL(pathname, `${getLinkedInBaseUrl(env)}/`).toString();
+}
+function toAbsoluteLinkedInUrl(value) {
+    const trimmed = normalizeWhitespace(value);
+    if (trimmed.length === 0) {
+        return undefined;
+    }
+    const url = trimmed.startsWith("http://") || trimmed.startsWith("https://")
+        ? new URL(trimmed)
+        : new URL(trimmed, `${getLinkedInBaseUrl()}/`);
+    if (!isAllowedLinkedInHostname(url.hostname)) {
+        return undefined;
+    }
+    url.search = "";
+    url.hash = "";
+    return url.toString().replace(/\/+$/, "");
+}
+function toAbsoluteUrl(value) {
+    const trimmed = normalizeWhitespace(value);
+    if (trimmed.length === 0) {
+        return undefined;
+    }
+    try {
+        return new URL(trimmed, `${getLinkedInBaseUrl()}/`).toString();
+    }
+    catch {
+        return undefined;
+    }
+}
+function getLinkedInPathSegments(url) {
+    const parsed = new URL(url);
+    return parsed.pathname.split("/").filter((segment) => segment.length > 0);
+}
+function getCompanyHandle(url) {
+    if (!url) {
+        return undefined;
+    }
+    const segments = getLinkedInPathSegments(url);
+    const companyIndex = segments.findIndex((segment) => segment.toLowerCase() === "company");
+    if (companyIndex === -1) {
+        return undefined;
+    }
+    const handle = normalizeWhitespace(segments[companyIndex + 1]);
+    return handle.length > 0 ? handle.toLowerCase() : undefined;
+}
+function getProductSlug(url) {
+    const segments = getLinkedInPathSegments(url);
+    const productsIndex = segments.findIndex((segment) => segment.toLowerCase() === "products");
+    const slug = normalizeWhitespace(segments[productsIndex + 1]);
+    if (productsIndex === -1 || slug.length === 0) {
+        throw new Error(`LinkedIn product URL is invalid: ${url}`);
+    }
+    return decodeURIComponent(slug).toLowerCase();
+}
+function getCategorySlug(url) {
+    const segments = getLinkedInPathSegments(url);
+    const categoriesIndex = segments.findIndex((segment) => segment.toLowerCase() === "categories");
+    const slug = normalizeWhitespace(segments[categoriesIndex + 1]);
+    if (categoriesIndex === -1 || slug.length === 0) {
+        throw new Error(`LinkedIn category URL is invalid: ${url}`);
+    }
+    return decodeURIComponent(slug).toLowerCase();
+}
+function parseResultCount(value) {
+    const match = normalizeWhitespace(value).match(/([\d,.]+)\s+results/i);
+    if (!match) {
+        return undefined;
+    }
+    const digits = match[1]?.replace(/[^\d]/g, "");
+    if (!digits) {
+        return undefined;
+    }
+    const parsed = Number(digits);
+    return Number.isFinite(parsed) ? parsed : undefined;
+}
+function parseCategoryCodeFromHtml($) {
+    const rawValue = $("#filterValues").html();
+    if (!rawValue) {
+        return undefined;
+    }
+    const normalized = rawValue.replace(/^<!--/, "").replace(/-->$/, "").trim();
+    if (normalized.length === 0) {
+        return undefined;
+    }
+    try {
+        const parsed = JSON.parse(normalized);
+        const currentUrn = parsed.currentCategoryUrn ?? parsed.productCategoryUrns?.[0];
+        const code = currentUrn?.split(":").at(-1)?.trim();
+        return code && code.length > 0 ? code : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function getImageUrl(element) {
+    const delayed = normalizeWhitespace(element.attr("data-delayed-url"));
+    if (delayed.length > 0) {
+        return toAbsoluteUrl(delayed);
+    }
+    const src = normalizeWhitespace(element.attr("src"));
+    return src.length > 0 ? toAbsoluteUrl(src) : undefined;
+}
+function buildCategoryFromUrl(url, name, options) {
+    return {
+        name,
+        slug: getCategorySlug(url),
+        url,
+        code: options?.code,
+        description: options?.description,
+        totalResults: options?.totalResults
+    };
+}
+function parseSubtitleForCategory(subtitle) {
+    const normalized = normalizeWhitespace(subtitle);
+    const match = normalized.match(/^(.*?)\s+by\s+(.*)$/i);
+    if (!match) {
+        return {};
+    }
+    return {
+        categoryName: normalizeWhitespace(match[1]),
+        vendorName: normalizeWhitespace(match[2])
+    };
+}
+function uniqueNonEmptyText(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        const normalized = normalizeWhitespace(value);
+        if (normalized.length === 0) {
+            continue;
+        }
+        if (seen.has(normalized)) {
+            continue;
+        }
+        seen.add(normalized);
+        result.push(normalized);
+    }
+    return result;
+}
+function pickLongerText(...values) {
+    const normalized = values
+        .map((value) => normalizeWhitespace(value))
+        .filter((value) => value.length > 0);
+    normalized.sort((left, right) => right.length - left.length);
+    return normalized[0];
+}
+function toRecordFromProductPage(detail, options) {
+    return {
+        productName: detail.productName,
+        productSlug: detail.productSlug,
+        productUrl: detail.productUrl,
+        imageUrl: detail.imageUrl,
+        description: detail.description,
+        vendor: detail.vendor,
+        category: detail.category,
+        pageNumber: options?.pageNumber ?? 1,
+        positionOnPage: options?.positionOnPage ?? 1,
+        learnMoreUrl: detail.learnMoreUrl,
+        intendedRoles: detail.intendedRoles,
+        usedBy: detail.usedBy,
+        rawPayload: options?.rawPayload ?? {
+            source: "product-page"
+        }
+    };
+}
+function parseProductCard($, card, pageNumber, positionOnPage, fallbackCategory) {
+    const productAnchor = card.find("h3 a, .product-serp-card__image-container").first();
+    const productUrl = toAbsoluteLinkedInUrl(productAnchor.attr("href"));
+    const productName = normalizeWhitespace(card.find("h3").text());
+    if (!productUrl || productName.length === 0) {
+        return null;
+    }
+    const subtitle = normalizeWhitespace(card.find("h4").text());
+    const subtitleParts = parseSubtitleForCategory(subtitle);
+    const vendorAnchor = card.find("h4 a").first();
+    const vendorName = normalizeWhitespace(vendorAnchor.text()) || subtitleParts.vendorName;
+    const vendorCompanyUrl = toAbsoluteLinkedInUrl(vendorAnchor.attr("href"));
+    const vendorHandle = getCompanyHandle(vendorCompanyUrl);
+    const categoryName = subtitleParts.categoryName ?? fallbackCategory?.name;
+    if (!categoryName || categoryName.length === 0) {
+        return null;
+    }
+    const category = fallbackCategory ?? {
+        name: categoryName,
+        slug: categoryName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""),
+        url: ""
+    };
+    return {
+        productName,
+        productSlug: getProductSlug(productUrl),
+        productUrl,
+        imageUrl: getImageUrl(card.find("img").first()),
+        description: normalizeWhitespace(card.find(".product-serp-card__description").text()) || undefined,
+        vendor: vendorName
+            ? {
+                name: vendorName,
+                companyUrl: vendorCompanyUrl,
+                handle: vendorHandle
+            }
+            : undefined,
+        category,
+        pageNumber,
+        positionOnPage,
+        intendedRoles: [],
+        usedBy: [],
+        rawPayload: {
+            subtitle,
+            source: "category-card"
+        }
+    };
+}
+export function parseLinkedInCategoryPage(html, requestUrl) {
+    const $ = load(html);
+    const canonicalUrl = toAbsoluteLinkedInUrl($('link[rel="canonical"]').attr("href") ?? requestUrl);
+    if (!canonicalUrl || !canonicalUrl.includes("/products/categories/")) {
+        throw new Error("LinkedIn category page did not expose a canonical category URL.");
+    }
+    const categoryName = normalizeWhitespace($(".serp-hero__title").text())
+        .replace(/^Find top products in\s+/i, "")
+        .replace(/\s+category$/i, "");
+    const categoryDescription = normalizeWhitespace($(".serp-hero__subtitle").text()) || undefined;
+    const totalResults = parseResultCount($(".serp-hero__results-title").text());
+    const categoryCode = parseCategoryCodeFromHtml($);
+    const category = buildCategoryFromUrl(canonicalUrl, categoryName, {
+        code: categoryCode,
+        description: categoryDescription,
+        totalResults
+    });
+    const items = [];
+    $('[data-product-cards-list] > li').each((index, element) => {
+        const parsed = parseProductCard($, $(element), getRequestedPageNumber(requestUrl), index + 1, category);
+        if (parsed) {
+            items.push(parsed);
+        }
+    });
+    return {
+        category,
+        items
+    };
+}
+export function parseLinkedInProductPage(html, requestUrl) {
+    const $ = load(html);
+    const canonicalUrl = toAbsoluteLinkedInUrl($('link[rel="canonical"]').attr("href") ?? requestUrl);
+    if (!canonicalUrl || !canonicalUrl.includes("/products/")) {
+        throw new Error("LinkedIn product page did not expose a canonical product URL.");
+    }
+    const productName = normalizeWhitespace($("h1.top-card-layout__title").first().text());
+    const categoryAnchor = $('[data-tracking-control-name="products_details_guest_product_category"]').first();
+    const categoryUrl = toAbsoluteLinkedInUrl(categoryAnchor.attr("href"));
+    const categoryName = normalizeWhitespace(categoryAnchor.text());
+    if (!categoryUrl || categoryName.length === 0) {
+        throw new Error("LinkedIn product page did not expose a product category.");
+    }
+    const category = buildCategoryFromUrl(categoryUrl, categoryName);
+    const vendorAnchor = $('[data-tracking-control-name="products_details_guest_organization_page"]').first();
+    const vendorCompanyUrl = toAbsoluteLinkedInUrl(vendorAnchor.attr("href"));
+    const vendorName = normalizeWhitespace(vendorAnchor.text());
+    const usedBy = [];
+    $(".customer__organization-name").each((_, element) => {
+        const anchor = $(element);
+        const name = normalizeWhitespace(anchor.text());
+        if (!name) {
+            return;
+        }
+        const container = anchor.closest("li, .customer");
+        usedBy.push({
+            name,
+            companyUrl: toAbsoluteLinkedInUrl(anchor.attr("href")),
+            logoUrl: getImageUrl(container.find("img").first())
+        });
+    });
+    const metaDescription = normalizeWhitespace($('meta[name="description"]').attr("content"));
+    const cleanedMetaDescription = metaDescription.startsWith(`${productName} |`)
+        ? normalizeWhitespace(metaDescription.slice(productName.length + 2))
+        : metaDescription;
+    const aboutDescription = normalizeWhitespace($(".about > p")
+        .not(".about__roles-title")
+        .first()
+        .text());
+    const intendedRoles = uniqueNonEmptyText($(".about__roles-item")
+        .map((_, element) => $(element).text())
+        .get());
+    return {
+        productName,
+        productSlug: getProductSlug(canonicalUrl),
+        productUrl: canonicalUrl,
+        imageUrl: getImageUrl($("img.top-card-layout__entity-image").first()),
+        description: pickLongerText(aboutDescription, cleanedMetaDescription),
+        category,
+        vendor: vendorName
+            ? {
+                name: vendorName,
+                companyUrl: vendorCompanyUrl,
+                handle: getCompanyHandle(vendorCompanyUrl)
+            }
+            : undefined,
+        learnMoreUrl: toAbsoluteUrl($(".top-card-layout__cta--secondary").attr("href")),
+        intendedRoles,
+        usedBy,
+    };
+}
+export function parseLinkedInCompanyPageForMainProduct(html, requestUrl) {
+    const $ = load(html);
+    const categoryAnchor = $('[data-tracking-control-name="organization_guest_main_product_card_category_link"]').first();
+    const productAnchor = $('[data-tracking-control-name="organization_guest_main_product_card"]').first();
+    const productUrl = toAbsoluteLinkedInUrl(productAnchor.attr("href"));
+    const categoryUrl = toAbsoluteLinkedInUrl(categoryAnchor.attr("href"));
+    const categoryName = normalizeWhitespace(categoryAnchor.text());
+    const productName = normalizeWhitespace(categoryAnchor.closest(".base-main-card").find(".base-main-card__title").first().text());
+    const companyUrl = toAbsoluteLinkedInUrl(requestUrl) ?? requestUrl;
+    if (!productUrl || !categoryUrl || categoryName.length === 0 || productName.length === 0) {
+        throw new Error("LinkedIn company page did not expose a main product card.");
+    }
+    return {
+        productName,
+        productUrl,
+        companyUrl,
+        category: buildCategoryFromUrl(categoryUrl, categoryName),
+        description: normalizeWhitespace(categoryAnchor.closest(".base-main-card").find(".base-main-card__description").text()) || undefined
+    };
+}
+export function parseLinkedInProductSearchPage(html, requestUrl) {
+    const $ = load(html);
+    const query = normalizeWhitespace($(".serp-hero__title").text()).replace(/^Find top products in\s+/i, "").replace(/^"|"$/g, "");
+    const items = [];
+    $('[data-product-cards-list] > li').each((index, element) => {
+        const parsed = parseProductCard($, $(element), getRequestedPageNumber(requestUrl), index + 1);
+        if (parsed) {
+            items.push(parsed);
+        }
+    });
+    return { query: query || undefined, items };
+}
+function normalizeQueryKey(value) {
+    return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+function pickBestSearchMatch(items, query) {
+    const queryKey = normalizeQueryKey(query);
+    if (queryKey.length === 0) {
+        const first = items[0];
+        if (!first) {
+            throw new Error("LinkedIn product search did not return any products.");
+        }
+        return first;
+    }
+    const scored = items.map((item) => {
+        const productKey = normalizeQueryKey(item.productName);
+        const productSlugKey = normalizeQueryKey(item.productSlug);
+        const vendorKey = normalizeQueryKey(item.vendor?.name ?? "");
+        const vendorHandleKey = normalizeQueryKey(item.vendor?.handle ?? "");
+        let score = 0;
+        for (const candidate of [productKey, productSlugKey, vendorKey, vendorHandleKey]) {
+            if (candidate.length === 0) {
+                continue;
+            }
+            if (candidate === queryKey) {
+                score += 100;
+            }
+            else if (candidate.startsWith(queryKey)) {
+                score += 40;
+            }
+            else if (candidate.includes(queryKey)) {
+                score += 20;
+            }
+            if (queryKey.startsWith(candidate) && candidate.length > 1) {
+                score += 15;
+            }
+        }
+        return { item, score };
+    });
+    scored.sort((left, right) => right.score - left.score);
+    return scored[0]?.item ?? items[0];
+}
+function classifyLinkedInInput(input) {
+    const trimmed = normalizeWhitespace(input);
+    if (trimmed.length === 0) {
+        throw new Error("A company domain or LinkedIn URL is required.");
+    }
+    const isUrl = /^https?:\/\//i.test(trimmed);
+    if (!isUrl) {
+        const domain = normalizeDomainInput(trimmed);
+        if (domain.length === 0) {
+            throw new Error("Could not understand the input. Pass a domain or LinkedIn URL.");
+        }
+        return {
+            kind: "domain",
+            input: trimmed,
+            query: domain.split(".")[0] ?? domain
+        };
+    }
+    const parsed = new URL(trimmed);
+    const hostname = parsed.hostname.toLowerCase();
+    if (!/(^|\.)linkedin\.com$/.test(hostname)) {
+        const domain = normalizeDomainInput(trimmed);
+        return {
+            kind: "domain",
+            input: trimmed,
+            query: domain.split(".")[0] ?? domain
+        };
+    }
+    if (parsed.pathname.includes("/search/results/products")) {
+        const privateCategory = parsePrivateCategoryCode(parsed.searchParams.get("productCategory"));
+        if (privateCategory) {
+            return {
+                kind: "private-category-code",
+                input: trimmed,
+                categoryCode: privateCategory
+            };
+        }
+        const query = normalizeWhitespace(parsed.searchParams.get("q"));
+        if (query) {
+            return {
+                kind: "search-query",
+                input: trimmed,
+                url: buildLinkedInUrl(`/products/search/?q=${encodeURIComponent(query)}`),
+                query
+            };
+        }
+        throw new Error("LinkedIn private product search URLs need a category code or query.");
+    }
+    if (parsed.pathname.includes("/products/search")) {
+        const query = normalizeWhitespace(parsed.searchParams.get("q") ?? parsed.searchParams.get("keywords"));
+        return {
+            kind: "search-query",
+            input: trimmed,
+            url: buildLinkedInUrl(`/products/search/?q=${encodeURIComponent(query)}`),
+            query
+        };
+    }
+    if (parsed.pathname.includes("/products/categories/")) {
+        return {
+            kind: "category-url",
+            input: trimmed,
+            url: toAbsoluteLinkedInUrl(trimmed) ?? trimmed
+        };
+    }
+    if (parsed.pathname.includes("/products/")) {
+        return {
+            kind: "product-url",
+            input: trimmed,
+            url: toAbsoluteLinkedInUrl(trimmed) ?? trimmed
+        };
+    }
+    if (parsed.pathname.includes("/company/")) {
+        return {
+            kind: "company-url",
+            input: trimmed,
+            url: toAbsoluteLinkedInUrl(trimmed) ?? trimmed
+        };
+    }
+    throw new Error("Pass a company domain, LinkedIn company page, LinkedIn product page, or LinkedIn category URL.");
+}
+function parsePrivateCategoryCode(value) {
+    const normalized = normalizeWhitespace(value);
+    if (normalized.length === 0) {
+        return undefined;
+    }
+    try {
+        const parsed = JSON.parse(normalized);
+        if (Array.isArray(parsed)) {
+            const first = normalizeWhitespace(String(parsed[0] ?? ""));
+            return first || undefined;
+        }
+    }
+    catch {
+        // Fall through to plain value parsing.
+    }
+    const match = normalized.match(/(\d{3,})/);
+    return match?.[1];
+}
+function getRequestedPageNumber(url) {
+    try {
+        const parsed = new URL(url);
+        const page = Number(parsed.searchParams.get("page") ?? "1");
+        return Number.isInteger(page) && page > 0 ? page : 1;
+    }
+    catch {
+        return 1;
+    }
+}
+function buildCategoryPageUrl(categoryUrl, pageNumber) {
+    const url = new URL(categoryUrl);
+    if (pageNumber <= 1) {
+        url.searchParams.delete("page");
+    }
+    else {
+        url.searchParams.set("page", String(pageNumber));
+    }
+    return url.toString();
+}
+export function createLinkedInHtmlFetcher(fetchImpl = fetch) {
+    return async (url) => {
+        const response = await fetchImpl(url, {
+            headers: {
+                "User-Agent": DEFAULT_LINKEDIN_SCRAPER_USER_AGENT,
+                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            },
+            redirect: "follow"
+        });
+        if (!response.ok) {
+            throw new Error(`LinkedIn request failed (${response.status}) for ${url}`);
+        }
+        return await response.text();
+    };
+}
+async function findCategoryByCode(categoryCode, fetchHtml) {
+    const browseHtml = await fetchHtml(buildLinkedInUrl("/products/categories/browse"));
+    const $ = load(browseHtml);
+    const urls = new Set();
+    $('a[href*="/products/categories/"]').each((_, element) => {
+        const absolute = toAbsoluteLinkedInUrl($(element).attr("href"));
+        if (absolute?.includes("/products/categories/")) {
+            urls.add(absolute);
+        }
+    });
+    for (const url of urls) {
+        try {
+            const html = await fetchHtml(url);
+            const parsed = parseLinkedInCategoryPage(html, url);
+            if (parsed.category.code === categoryCode) {
+                return parsed.category;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    throw new Error(`Could not resolve LinkedIn product category code ${categoryCode}.`);
+}
+export async function resolveLinkedInProductSource(input, fetchHtml) {
+    const resolvedInput = classifyLinkedInInput(input);
+    if (resolvedInput.kind === "category-url") {
+        const html = await fetchHtml(resolvedInput.url);
+        const page = parseLinkedInCategoryPage(html, resolvedInput.url);
+        return {
+            input,
+            kind: resolvedInput.kind,
+            category: page.category
+        };
+    }
+    if (resolvedInput.kind === "product-url") {
+        const html = await fetchHtml(resolvedInput.url);
+        const product = parseLinkedInProductPage(html, resolvedInput.url);
+        return {
+            input,
+            kind: resolvedInput.kind,
+            productUrl: product.productUrl,
+            matchedProductName: product.productName,
+            category: product.category
+        };
+    }
+    if (resolvedInput.kind === "company-url") {
+        const html = await fetchHtml(resolvedInput.url);
+        const reference = parseLinkedInCompanyPageForMainProduct(html, resolvedInput.url);
+        return {
+            input,
+            kind: resolvedInput.kind,
+            companyUrl: reference.companyUrl,
+            productUrl: reference.productUrl,
+            matchedProductName: reference.productName,
+            category: reference.category
+        };
+    }
+    if (resolvedInput.kind === "private-category-code") {
+        const category = await findCategoryByCode(resolvedInput.categoryCode, fetchHtml);
+        return {
+            input,
+            kind: resolvedInput.kind,
+            category: {
+                ...category,
+                code: resolvedInput.categoryCode
+            }
+        };
+    }
+    const searchQuery = resolvedInput.query ?? "";
+    const searchUrl = resolvedInput.kind === "domain"
+        ? buildLinkedInUrl(`/products/search/?q=${encodeURIComponent(searchQuery)}`)
+        : resolvedInput.url;
+    const html = await fetchHtml(searchUrl);
+    const search = parseLinkedInProductSearchPage(html, searchUrl);
+    const rankedCandidates = (() => {
+        const best = pickBestSearchMatch(search.items, searchQuery);
+        const seen = new Set();
+        const ordered = [];
+        for (const candidate of [best, ...search.items]) {
+            if (!candidate || seen.has(candidate.productUrl)) {
+                continue;
+            }
+            seen.add(candidate.productUrl);
+            ordered.push(candidate);
+        }
+        return ordered;
+    })();
+    let product = null;
+    let matched = null;
+    let lastError = null;
+    for (const candidate of rankedCandidates) {
+        try {
+            const productHtml = await fetchHtml(candidate.productUrl);
+            product = parseLinkedInProductPage(productHtml, candidate.productUrl);
+            matched = candidate;
+            break;
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error.message : String(error);
+        }
+    }
+    if (!product || !matched) {
+        throw new Error(lastError
+            ? `Could not resolve a LinkedIn product page from search results. Last error: ${lastError}`
+            : "Could not resolve a LinkedIn product page from search results.");
+    }
+    return {
+        input,
+        kind: resolvedInput.kind,
+        query: searchQuery,
+        productUrl: product.productUrl,
+        matchedProductName: product.productName,
+        category: product.category
+    };
+}
+function mergeDetailIntoRecord(record, detail) {
+    const description = record.description && record.description.length >= (detail.description?.length ?? 0)
+        ? record.description
+        : detail.description;
+    return {
+        ...record,
+        imageUrl: record.imageUrl ?? detail.imageUrl,
+        description,
+        vendor: record.vendor ?? detail.vendor,
+        category: {
+            ...record.category,
+            code: record.category.code ?? detail.category.code,
+            description: record.category.description ?? detail.category.description,
+            totalResults: record.category.totalResults ?? detail.category.totalResults
+        },
+        learnMoreUrl: detail.learnMoreUrl ?? record.learnMoreUrl,
+        intendedRoles: detail.intendedRoles.length > 0 ? detail.intendedRoles : record.intendedRoles,
+        usedBy: detail.usedBy.length > 0 ? detail.usedBy : record.usedBy,
+        rawPayload: {
+            ...record.rawPayload,
+            detailSource: "product-page"
+        }
+    };
+}
+async function mapWithConcurrency(input, concurrency, mapper) {
+    if (input.length === 0) {
+        return [];
+    }
+    const safeConcurrency = Math.max(1, Math.min(concurrency, input.length));
+    const results = new Array(input.length);
+    let nextIndex = 0;
+    const worker = async () => {
+        while (true) {
+            const current = nextIndex;
+            nextIndex += 1;
+            if (current >= input.length) {
+                return;
+            }
+            results[current] = await mapper(input[current], current);
+        }
+    };
+    await Promise.all(Array.from({ length: safeConcurrency }, () => worker()));
+    return results;
+}
+export async function crawlLinkedInProductCategory(options) {
+    const fetchHtml = options.fetchHtml ?? createLinkedInHtmlFetcher();
+    const source = await resolveLinkedInProductSource(options.input, fetchHtml);
+    const maxPages = Math.max(1, options.maxPages ?? 25);
+    const limit = options.limit !== undefined ? Math.max(1, options.limit) : undefined;
+    const itemsByUrl = new Map();
+    let totalPagesFetched = 0;
+    if (options.enrichDetails !== false && source.productUrl) {
+        try {
+            const sourceDetailHtml = await fetchHtml(source.productUrl);
+            const sourceDetail = parseLinkedInProductPage(sourceDetailHtml, source.productUrl);
+            itemsByUrl.set(sourceDetail.productUrl, toRecordFromProductPage(sourceDetail, {
+                rawPayload: {
+                    source: "resolved-product-page",
+                    resolvedFromInput: true
+                }
+            }));
+        }
+        catch {
+            // Keep crawling the category even if the resolved source product detail page is temporarily unavailable.
+        }
+    }
+    for (let pageNumber = 1; pageNumber <= maxPages; pageNumber += 1) {
+        const pageUrl = buildCategoryPageUrl(source.category.url, pageNumber);
+        const html = await fetchHtml(pageUrl);
+        const page = parseLinkedInCategoryPage(html, pageUrl);
+        totalPagesFetched = pageNumber;
+        if (!source.category.code && page.category.code) {
+            source.category.code = page.category.code;
+        }
+        if (!source.category.description && page.category.description) {
+            source.category.description = page.category.description;
+        }
+        if (!source.category.totalResults && page.category.totalResults) {
+            source.category.totalResults = page.category.totalResults;
+        }
+        if (limit !== undefined && itemsByUrl.size >= limit) {
+            break;
+        }
+        if (page.items.length === 0) {
+            break;
+        }
+        let newItemsOnPage = 0;
+        for (const item of page.items) {
+            if (!itemsByUrl.has(item.productUrl)) {
+                itemsByUrl.set(item.productUrl, item);
+                newItemsOnPage += 1;
+            }
+            if (limit !== undefined && itemsByUrl.size >= limit) {
+                break;
+            }
+        }
+        if (limit !== undefined && itemsByUrl.size >= limit) {
+            break;
+        }
+        if (newItemsOnPage === 0) {
+            break;
+        }
+    }
+    let items = Array.from(itemsByUrl.values());
+    if (limit !== undefined) {
+        items = items.slice(0, limit);
+    }
+    if (options.enrichDetails !== false && items.length > 0) {
+        const detailed = await mapWithConcurrency(items, options.detailConcurrency ?? 4, async (item) => {
+            try {
+                const html = await fetchHtml(item.productUrl);
+                const detail = parseLinkedInProductPage(html, item.productUrl);
+                return mergeDetailIntoRecord(item, detail);
+            }
+            catch (error) {
+                return {
+                    ...item,
+                    rawPayload: {
+                        ...(item.rawPayload && typeof item.rawPayload === "object" ? item.rawPayload : {}),
+                        detailFetchError: error instanceof Error ? error.message : String(error)
+                    }
+                };
+            }
+        });
+        items = detailed;
+    }
+    return {
+        source,
+        items,
+        totalPagesFetched
+    };
+}