indiecrm-cli 0.1.0 → 0.2.1

package/dist/linkedin-companies.js

@@ -0,0 +1,562 @@
+import { load } from "cheerio";
+import { BigQuery } from "@google-cloud/bigquery";
+import { DEFAULT_LINKEDIN_SCRAPER_USER_AGENT } from "./linkedin-session-contracts.js";
+const DEFAULT_LINKEDIN_BASE_URL = "https://www.linkedin.com";
+const DEFAULT_RAPIDAPI_LINKEDIN_COMPANY_HOST = "web-scraping-api2.p.rapidapi.com";
+const DEFAULT_RAPIDAPI_LINKEDIN_COMPANY_ENDPOINT = "https://web-scraping-api2.p.rapidapi.com/get-company-by-linkedinurl";
+const DEFAULT_BIGQUERY_PROJECT_ID = process.env.BQ_PROJECT_ID ?? process.env.GOOGLE_CLOUD_PROJECT ?? process.env.GCLOUD_PROJECT ?? "icpidentifier";
+const LINKEDIN_COMPANIES_TABLE = "`icpidentifier.SalesGPT.linkedin_companies_processed`";
+const LEADPOOL_TABLE = "`icpidentifier.SalesGPT.leadPool_new`";
+function normalizeWhitespace(value) {
+    return (value ?? "").replace(/\s+/g, " ").trim();
+}
+function getLinkedInBaseUrl(env = process.env) {
+    const override = env.SALESPROMPTER_LINKEDIN_BASE_URL?.trim();
+    if (!override) {
+        return DEFAULT_LINKEDIN_BASE_URL;
+    }
+    return override.replace(/\/+$/, "");
+}
+function buildLinkedInCompanyUrl(companyId, env = process.env) {
+    return new URL(`/company/${companyId}/`, `${getLinkedInBaseUrl(env)}/`).toString().replace(/\/+$/, "");
+}
+function getCompanyHandleFromUrl(url) {
+    if (!url) {
+        return undefined;
+    }
+    const pathname = new URL(url).pathname;
+    const segments = pathname.split("/").filter(Boolean);
+    const companyIndex = segments.findIndex((segment) => segment.toLowerCase() === "company");
+    if (companyIndex === -1) {
+        return undefined;
+    }
+    const handle = normalizeWhitespace(segments[companyIndex + 1]);
+    if (!handle || /^\d+$/.test(handle)) {
+        return undefined;
+    }
+    return handle.toLowerCase();
+}
+function extractDomainFromWebsite(website) {
+    if (!website) {
+        return undefined;
+    }
+    try {
+        return new URL(website).hostname.replace(/^www\./i, "").toLowerCase();
+    }
+    catch {
+        return undefined;
+    }
+}
+function getRapidApiLinkedInCompanyConfig(env = process.env) {
+    const apiKey = env.RAPIDAPI_KEY?.trim();
+    if (!apiKey) {
+        return null;
+    }
+    return {
+        apiKey,
+        host: env.RAPIDAPI_LINKEDIN_COMPANY_HOST?.trim() || DEFAULT_RAPIDAPI_LINKEDIN_COMPANY_HOST,
+        endpoint: env.RAPIDAPI_LINKEDIN_COMPANY_ENDPOINT?.trim() || DEFAULT_RAPIDAPI_LINKEDIN_COMPANY_ENDPOINT
+    };
+}
+function parseJsonLdOrganization($) {
+    const scripts = $('script[type="application/ld+json"]')
+        .map((_, element) => $(element).text())
+        .get();
+    for (const rawText of scripts) {
+        const text = rawText.trim();
+        if (!text) {
+            continue;
+        }
+        try {
+            const parsed = JSON.parse(text);
+            const items = Array.isArray(parsed) ? parsed : [parsed];
+            for (const item of items) {
+                if (!item || typeof item !== "object") {
+                    continue;
+                }
+                const candidate = item;
+                const type = normalizeWhitespace(String(candidate["@type"] ?? ""));
+                if (type.toLowerCase() === "organization") {
+                    return candidate;
+                }
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+function readDefinitionMap($) {
+    const entries = new Map();
+    $("dt").each((_, element) => {
+        const term = normalizeWhitespace($(element).text()).toLowerCase();
+        const description = normalizeWhitespace($(element).next("dd").text());
+        if (term && description) {
+            entries.set(term, description);
+        }
+    });
+    return entries;
+}
+function parseEmployeeCount(value) {
+    const digits = normalizeWhitespace(value).replace(/[^\d]/g, "");
+    if (!digits) {
+        return undefined;
+    }
+    const parsed = Number(digits);
+    return Number.isFinite(parsed) ? parsed : undefined;
+}
+function parseFoundedYear(value) {
+    const match = normalizeWhitespace(value).match(/\b(19|20)\d{2}\b/);
+    if (!match) {
+        return undefined;
+    }
+    return Number(match[0]);
+}
+export function parseLinkedInCompanyPage(html, requestUrl) {
+    const $ = load(html);
+    const timestamp = new Date().toISOString();
+    const canonicalUrl = normalizeWhitespace($('link[rel="canonical"]').attr("href")) || requestUrl;
+    const unavailableTitle = normalizeWhitespace($("title").text()).toLowerCase();
+    const unavailableBody = normalizeWhitespace($("body").text()).toLowerCase();
+    const unavailable = unavailableTitle.includes("page not found") ||
+        unavailableTitle.includes("linkedin") && unavailableTitle.includes("unavailable") ||
+        unavailableBody.includes("page isn't available") ||
+        unavailableBody.includes("this page doesn't exist");
+    const jsonLd = parseJsonLdOrganization($);
+    const definitions = readDefinitionMap($);
+    const requestIdMatch = requestUrl.match(/\/company\/(\d+)(?:\/|$)/i);
+    const canonicalIdMatch = canonicalUrl.match(/\/company\/(\d+)(?:\/|$)/i);
+    const resolvedId = Number(canonicalIdMatch?.[1] ?? requestIdMatch?.[1]);
+    if (!Number.isFinite(resolvedId)) {
+        throw new Error(`LinkedIn company URL is missing a numeric company id: ${requestUrl}`);
+    }
+    const name = normalizeWhitespace(String(jsonLd?.name ?? "")) ||
+        normalizeWhitespace($("h1").first().text()) ||
+        normalizeWhitespace($('meta[property="og:title"]').attr("content")) ||
+        undefined;
+    const jsonLdUrl = normalizeWhitespace(String(jsonLd?.url ?? ""));
+    const jsonLdWebsite = (() => {
+        if (!jsonLdUrl)
+            return undefined;
+        try {
+            const hostname = new URL(jsonLdUrl).hostname;
+            return /(^|\.)linkedin\.com$/i.test(hostname) ? undefined : jsonLdUrl;
+        }
+        catch {
+            return undefined;
+        }
+    })();
+    const website = normalizeWhitespace(definitions.get("website")) ||
+        jsonLdWebsite ||
+        undefined;
+    const description = normalizeWhitespace(String(jsonLd?.description ?? "")) ||
+        normalizeWhitespace($('meta[name="description"]').attr("content")) ||
+        undefined;
+    const tagline = normalizeWhitespace($(".top-card-layout__headline").first().text()) ||
+        normalizeWhitespace($(".org-top-card-summary__tagline").first().text()) ||
+        undefined;
+    const industry = normalizeWhitespace(definitions.get("industry")) || undefined;
+    const companySize = normalizeWhitespace(definitions.get("company size")) || undefined;
+    const headquarters = normalizeWhitespace(definitions.get("headquarters")) || undefined;
+    const specialties = normalizeWhitespace(definitions.get("specialties")) || undefined;
+    const employeesOnLinkedIn = parseEmployeeCount(definitions.get("employees"));
+    const founded = parseFoundedYear(definitions.get("founded"));
+    return {
+        id: resolvedId,
+        handle: getCompanyHandleFromUrl(canonicalUrl),
+        name,
+        industry,
+        companySize,
+        headquarters,
+        website,
+        domain: extractDomainFromWebsite(website),
+        employeesOnLinkedIn,
+        description,
+        tagline,
+        specialties,
+        founded,
+        unavailable,
+        error: unavailable ? "LinkedIn company page is unavailable" : undefined,
+        timestamp
+    };
+}
+function parseRapidApiCompanyProfile(payload, candidate) {
+    const data = payload.data;
+    if (!data) {
+        throw new Error(`RapidAPI company response did not include data for ${candidate.companyUrl}`);
+    }
+    const id = Number(data.company_id ?? candidate.companyId);
+    if (!Number.isFinite(id)) {
+        throw new Error(`RapidAPI company response is missing a numeric company id for ${candidate.companyUrl}`);
+    }
+    const website = normalizeWhitespace(data.website) || undefined;
+    const linkedinUrl = normalizeWhitespace(data.linkedin_url) || candidate.companyUrl;
+    const industries = Array.isArray(data.industries) ? data.industries.map((value) => normalizeWhitespace(value)).filter(Boolean) : [];
+    return {
+        id,
+        handle: getCompanyHandleFromUrl(linkedinUrl),
+        name: normalizeWhitespace(data.company_name) || candidate.companyName,
+        industry: industries[0],
+        companySize: normalizeWhitespace(data.employee_range) || undefined,
+        headquarters: normalizeWhitespace(data.hq_full_address) || undefined,
+        website,
+        domain: normalizeWhitespace(data.domain) || extractDomainFromWebsite(website),
+        employeesOnLinkedIn: typeof data.employee_count === "number" && Number.isFinite(data.employee_count) ? data.employee_count : undefined,
+        description: normalizeWhitespace(data.description) || undefined,
+        tagline: normalizeWhitespace(data.tagline) || undefined,
+        specialties: normalizeWhitespace(data.specialties) || undefined,
+        founded: typeof data.year_founded === "number" && Number.isFinite(data.year_founded) ? data.year_founded : undefined,
+        unavailable: false,
+        timestamp: new Date().toISOString()
+    };
+}
+export function buildLinkedInCompanyBackfillSql(clientId, limit) {
+    return `WITH backlog AS (
+  SELECT
+    companyId,
+    ANY_VALUE(companyName) AS companyName,
+    ANY_VALUE(company_filter) AS companyFilter
+  FROM ${LEADPOOL_TABLE}
+  WHERE clientId = ${clientId}
+    AND companyId IS NOT NULL
+    AND (
+      COALESCE(company_toBeCrawled, FALSE) = TRUE
+      OR LOWER(CAST(company_filter AS STRING)) IN (
+        '02: company unavailable',
+        '02: linkedin companies unavailable',
+        '03: company to be crawled'
+      )
+    )
+  GROUP BY companyId
+)
+SELECT
+  companyId,
+  companyName,
+  companyFilter
+FROM backlog
+WHERE NOT EXISTS (
+  SELECT 1
+  FROM ${LINKEDIN_COMPANIES_TABLE} processed
+  WHERE processed.query = CONCAT('https://www.linkedin.com/company/', CAST(backlog.companyId AS STRING))
+)
+ORDER BY companyId
+LIMIT ${limit}`;
+}
+function sqlString(value) {
+    if (value === undefined) {
+        return "NULL";
+    }
+    return `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`;
+}
+function sqlInteger(value) {
+    return typeof value === "number" && Number.isFinite(value) ? String(Math.trunc(value)) : "NULL";
+}
+function sqlBoolean(value) {
+    return value ? "TRUE" : "FALSE";
+}
+export function buildLinkedInCompaniesMergeSql(rows) {
+    if (rows.length === 0) {
+        throw new Error("At least one LinkedIn company row is required for merge SQL.");
+    }
+    const sourceRows = rows
+        .map((row) => `STRUCT(
+  ${sqlInteger(row.id)} AS id,
+  ${sqlString(row.handle)} AS handle,
+  ${sqlString(row.name)} AS name,
+  NULL AS industryId,
+  ${sqlString(row.industry)} AS industry,
+  NULL AS industry_input,
+  NULL AS companySizeId,
+  ${sqlString(row.companySize)} AS companySize,
+  ${sqlString(row.headquarters)} AS headquarters,
+  NULL AS countryCode,
+  ${sqlString(row.website)} AS website_linkedin,
+  ${sqlString(row.domain)} AS domain_linkedin,
+  NULL AS hunter_emailCount,
+  NULL AS domainFinder_name,
+  NULL AS domain,
+  NULL AS emailDomainFinder_ts,
+  NULL AS company_missingHeadquarters,
+  FALSE AS blacklisted_bySalesPrompter,
+  FALSE AS company_countryCodeToBeProcessed,
+  FALSE AS domainBlacklisted,
+  FALSE AS company_emailDomainFinder_toBeProcessed,
+  FALSE AS company_emailDomainNotFound,
+  ${sqlInteger(row.employeesOnLinkedIn)} AS employeesOnLinkedIn,
+  NULL AS growth6Mth,
+  NULL AS growth1Yr,
+  NULL AS growth2Yr,
+  ${sqlString(row.description)} AS description,
+  ${sqlString(row.tagline)} AS tagline,
+  ${sqlString(row.specialties)} AS specialties,
+  ${sqlInteger(row.founded)} AS founded,
+  ${sqlString(row.error)} AS error,
+  ${sqlBoolean(row.unavailable)} AS linkedin_companies_companyUnavailable,
+  TIMESTAMP(${sqlString(row.timestamp)}) AS timestamp
+)`)
+        .join(",\n");
+    return `MERGE ${LINKEDIN_COMPANIES_TABLE} AS target
+USING (
+  SELECT *
+  FROM UNNEST([
+${sourceRows}
+  ])
+) AS source
+ON target.id = source.id
+WHEN MATCHED THEN UPDATE SET
+  handle = source.handle,
+  name = source.name,
+  industryId = source.industryId,
+  industry = source.industry,
+  industry_input = source.industry_input,
+  companySizeId = source.companySizeId,
+  companySize = source.companySize,
+  headquarters = source.headquarters,
+  countryCode = source.countryCode,
+  website_linkedin = source.website_linkedin,
+  domain_linkedin = source.domain_linkedin,
+  hunter_emailCount = source.hunter_emailCount,
+  domainFinder_name = source.domainFinder_name,
+  domain = source.domain,
+  emailDomainFinder_ts = source.emailDomainFinder_ts,
+  company_missingHeadquarters = source.company_missingHeadquarters,
+  blacklisted_bySalesPrompter = source.blacklisted_bySalesPrompter,
+  company_countryCodeToBeProcessed = source.company_countryCodeToBeProcessed,
+  domainBlacklisted = source.domainBlacklisted,
+  company_emailDomainFinder_toBeProcessed = source.company_emailDomainFinder_toBeProcessed,
+  company_emailDomainNotFound = source.company_emailDomainNotFound,
+  employeesOnLinkedIn = source.employeesOnLinkedIn,
+  growth6Mth = source.growth6Mth,
+  growth1Yr = source.growth1Yr,
+  growth2Yr = source.growth2Yr,
+  description = source.description,
+  tagline = source.tagline,
+  specialties = source.specialties,
+  founded = source.founded,
+  error = source.error,
+  linkedin_companies_companyUnavailable = source.linkedin_companies_companyUnavailable,
+  timestamp = source.timestamp
+WHEN NOT MATCHED THEN INSERT (
+  id,
+  handle,
+  name,
+  industryId,
+  industry,
+  industry_input,
+  companySizeId,
+  companySize,
+  headquarters,
+  countryCode,
+  website_linkedin,
+  domain_linkedin,
+  hunter_emailCount,
+  domainFinder_name,
+  domain,
+  emailDomainFinder_ts,
+  company_missingHeadquarters,
+  blacklisted_bySalesPrompter,
+  company_countryCodeToBeProcessed,
+  domainBlacklisted,
+  company_emailDomainFinder_toBeProcessed,
+  company_emailDomainNotFound,
+  employeesOnLinkedIn,
+  growth6Mth,
+  growth1Yr,
+  growth2Yr,
+  description,
+  tagline,
+  specialties,
+  founded,
+  error,
+  linkedin_companies_companyUnavailable,
+  timestamp
+) VALUES (
+  source.id,
+  source.handle,
+  source.name,
+  source.industryId,
+  source.industry,
+  source.industry_input,
+  source.companySizeId,
+  source.companySize,
+  source.headquarters,
+  source.countryCode,
+  source.website_linkedin,
+  source.domain_linkedin,
+  source.hunter_emailCount,
+  source.domainFinder_name,
+  source.domain,
+  source.emailDomainFinder_ts,
+  source.company_missingHeadquarters,
+  source.blacklisted_bySalesPrompter,
+  source.company_countryCodeToBeProcessed,
+  source.domainBlacklisted,
+  source.company_emailDomainFinder_toBeProcessed,
+  source.company_emailDomainNotFound,
+  source.employeesOnLinkedIn,
+  source.growth6Mth,
+  source.growth1Yr,
+  source.growth2Yr,
+  source.description,
+  source.tagline,
+  source.specialties,
+  source.founded,
+  source.error,
+  source.linkedin_companies_companyUnavailable,
+  source.timestamp
+)`;
+}
+export function createLinkedInCompaniesBigQueryClient(env = process.env) {
+    const credentialsJson = env.GOOGLE_SERVICE_ACCOUNT_KEY?.trim();
+    if (credentialsJson) {
+        const normalizedCredentialsJson = credentialsJson.replace(/"private_key":"([\s\S]*?)"/, (_, privateKey) => `"private_key":"${privateKey.replace(/\n/g, "\\n")}"`);
+        return new BigQuery({
+            projectId: DEFAULT_BIGQUERY_PROJECT_ID,
+            credentials: JSON.parse(normalizedCredentialsJson)
+        });
+    }
+    return new BigQuery({
+        projectId: DEFAULT_BIGQUERY_PROJECT_ID
+    });
+}
+export async function fetchLinkedInCompanyBackfillCandidates(bigQuery, clientId, limit) {
+    const [rows] = await bigQuery.query({
+        query: buildLinkedInCompanyBackfillSql(clientId, limit),
+        useLegacySql: false
+    });
+    return rows.map((row) => {
+        const companyId = Number(row.companyId);
+        if (!Number.isFinite(companyId)) {
+            throw new Error(`BigQuery backlog row is missing a numeric companyId: ${JSON.stringify(row)}`);
+        }
+        return {
+            companyId,
+            companyName: normalizeWhitespace(String(row.companyName ?? "")) || undefined,
+            companyFilter: normalizeWhitespace(String(row.companyFilter ?? "")) || undefined,
+            companyUrl: buildLinkedInCompanyUrl(companyId)
+        };
+    });
+}
+async function fetchHtml(url) {
+    const response = await fetch(url, {
+        headers: {
+            "User-Agent": DEFAULT_LINKEDIN_SCRAPER_USER_AGENT
+        }
+    });
+    return {
+        status: response.status,
+        html: await response.text()
+    };
+}
+export async function scrapeLinkedInCompany(candidate) {
+    const rapidApiConfig = getRapidApiLinkedInCompanyConfig();
+    if (rapidApiConfig) {
+        const url = new URL(rapidApiConfig.endpoint);
+        url.searchParams.set("linkedin_url", candidate.companyUrl);
+        const response = await fetch(url, {
+            headers: {
+                "x-rapidapi-key": rapidApiConfig.apiKey,
+                "x-rapidapi-host": rapidApiConfig.host,
+                "User-Agent": DEFAULT_LINKEDIN_SCRAPER_USER_AGENT
+            }
+        });
+        const text = await response.text();
+        const parsed = (text ? JSON.parse(text) : {});
+        if (response.status === 404) {
+            return {
+                id: candidate.companyId,
+                name: candidate.companyName,
+                unavailable: true,
+                error: `RapidAPI company endpoint returned 404 for ${candidate.companyUrl}`,
+                timestamp: new Date().toISOString()
+            };
+        }
+        if (!response.ok) {
+            throw new Error(`RapidAPI company endpoint returned ${response.status} for ${candidate.companyUrl}`);
+        }
+        return parseRapidApiCompanyProfile(parsed, candidate);
+    }
+    const response = await fetchHtml(candidate.companyUrl);
+    if (response.status === 404 || response.status === 410) {
+        return {
+            id: candidate.companyId,
+            name: candidate.companyName,
+            unavailable: true,
+            error: `LinkedIn company page returned ${response.status}`,
+            timestamp: new Date().toISOString()
+        };
+    }
+    if (response.status >= 400) {
+        throw new Error(`LinkedIn company page returned ${response.status} for ${candidate.companyUrl}`);
+    }
+    const parsed = parseLinkedInCompanyPage(response.html, candidate.companyUrl);
+    return {
+        ...parsed,
+        id: candidate.companyId,
+        name: parsed.name ?? candidate.companyName
+    };
+}
+async function mapWithConcurrency(values, concurrency, mapper) {
+    const results = new Array(values.length);
+    let currentIndex = 0;
+    async function worker() {
+        while (currentIndex < values.length) {
+            const index = currentIndex;
+            currentIndex += 1;
+            results[index] = await mapper(values[index]);
+        }
+    }
+    const workers = Array.from({ length: Math.max(1, Math.min(concurrency, values.length || 1)) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+export async function backfillLinkedInCompanies(options) {
+    const limit = options.limit ?? 25;
+    const concurrency = options.concurrency ?? 4;
+    const bigQuery = createLinkedInCompaniesBigQueryClient();
+    const candidates = await fetchLinkedInCompanyBackfillCandidates(bigQuery, options.clientId, limit);
+    const attempts = await mapWithConcurrency(candidates, concurrency, async (candidate) => {
+        try {
+            return {
+                candidate,
+                ok: true,
+                profile: await scrapeLinkedInCompany(candidate)
+            };
+        }
+        catch (error) {
+            return {
+                candidate,
+                ok: false,
+                error: error instanceof Error ? error.message : "LinkedIn company scrape failed"
+            };
+        }
+    });
+    const profiles = attempts.filter((attempt) => attempt.ok)
+        .map((attempt) => attempt.profile);
+    const results = attempts.map((attempt) => ({
+        companyId: attempt.candidate.companyId,
+        companyUrl: attempt.candidate.companyUrl,
+        companyName: attempt.ok ? attempt.profile.name : attempt.candidate.companyName,
+        unavailable: attempt.ok ? attempt.profile.unavailable : false,
+        error: attempt.ok ? attempt.profile.error : attempt.error
+    }));
+    if (options.dryRun || profiles.length === 0) {
+        return {
+            clientId: options.clientId,
+            candidates,
+            results,
+            mergeSql: profiles.length > 0 ? buildLinkedInCompaniesMergeSql(profiles) : undefined
+        };
+    }
+    const mergeSql = buildLinkedInCompaniesMergeSql(profiles);
+    await bigQuery.query({
+        query: mergeSql,
+        useLegacySql: false
+    });
+    return {
+        clientId: options.clientId,
+        candidates,
+        results
+    };
+}