autokap 1.4.2 → 1.5.1

package/dist/crm/scrape-betalist.d.ts

@@ -1,20 +0,0 @@
-export type BetaListLaunch = {
-    sourceUrl: string;
-    productName: string;
-    productUrl: string | null;
-    creatorName: string | null;
-    creatorEmail: string | null;
-    creatorHandle: string | null;
-    creatorLang: string | null;
-    tagline?: string | null;
-};
-export interface ScrapeBetaListOptions {
-    lookbackDays: number;
-    userAgent: string;
-    logger: {
-        info(msg: string): void;
-        warn(msg: string): void;
-        error(msg: string): void;
-    };
-}
-export declare function scrapeBetaListLaunches(opts: ScrapeBetaListOptions): Promise<BetaListLaunch[]>;

package/dist/crm/scrape-betalist.js

@@ -1,194 +0,0 @@
-const BETALIST_ORIGIN = 'https://betalist.com/';
-export async function scrapeBetaListLaunches(opts) {
-    let html;
-    try {
-        const response = await fetch(BETALIST_ORIGIN, {
-            headers: {
-                Accept: 'text/html',
-                'User-Agent': opts.userAgent,
-            },
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!response.ok) {
-            opts.logger.warn(`[crm-betalist] BetaList fetch returned HTTP ${response.status}`);
-            return [];
-        }
-        html = await response.text();
-    }
-    catch (error) {
-        opts.logger.warn(`[crm-betalist] BetaList fetch failed: ${error.message}`);
-        return [];
-    }
-    const cheerio = await loadCheerio();
-    const $ = cheerio.load(html);
-    const launches = [];
-    const seen = new Set();
-    $('a[href^="/startups/"], a[href*="betalist.com/startups/"]').each((_, anchor) => {
-        const href = $(anchor).attr('href');
-        if (!href)
-            return;
-        const sourceUrl = canonicalBetaListUrl(href);
-        if (!sourceUrl || seen.has(sourceUrl))
-            return;
-        const container = closestLaunchContainer($, anchor);
-        const productName = extractProductName($, anchor, container);
-        if (!productName) {
-            opts.logger.warn(`[crm-betalist] Missing product name for ${sourceUrl}`);
-            return;
-        }
-        const launchDate = parseLaunchDateForRow($, container);
-        if (launchDate && !isWithinLookback(launchDate, opts.lookbackDays))
-            return;
-        const productUrl = extractProductUrl($, container, sourceUrl);
-        const text = container.text().replace(/\s+/g, ' ').trim();
-        const creatorHandle = extractHandleFromHtml(container.html() ?? '');
-        launches.push({
-            sourceUrl,
-            productName,
-            productUrl,
-            creatorName: extractCreatorName($, container),
-            creatorEmail: extractEmail(text),
-            creatorHandle,
-            creatorLang: null,
-            tagline: extractTagline($, container, productName),
-        });
-        seen.add(sourceUrl);
-    });
-    if (launches.length === 0) {
-        opts.logger.warn('[crm-betalist] No BetaList launch links found on homepage');
-    }
-    return launches;
-}
-function canonicalBetaListUrl(href) {
-    try {
-        const url = new URL(href, BETALIST_ORIGIN);
-        if (url.hostname !== 'betalist.com' && url.hostname !== 'www.betalist.com')
-            return null;
-        if (!url.pathname.startsWith('/startups/'))
-            return null;
-        url.hash = '';
-        url.search = '';
-        return url.toString();
-    }
-    catch {
-        return null;
-    }
-}
-async function loadCheerio() {
-    const importer = new Function('specifier', 'return import(specifier)');
-    return importer('cheerio');
-}
-function closestLaunchContainer($, anchor) {
-    const candidates = $(anchor).parents('article, li, tr, div, section').toArray();
-    for (const candidate of candidates) {
-        const row = $(candidate);
-        const text = row.text().replace(/\s+/g, ' ').trim();
-        if (row.find('a[href^="/startups/"], a[href*="betalist.com/startups/"]').length >= 1 && text.length > 0) {
-            return row;
-        }
-    }
-    return $(anchor).parent();
-}
-function extractProductName($, anchor, container) {
-    const direct = $(anchor).text().replace(/\s+/g, ' ').trim();
-    if (direct && direct.length <= 200)
-        return direct;
-    const heading = container.find('h1, h2, h3, h4, [class*="title"], [class*="name"]').first().text().replace(/\s+/g, ' ').trim();
-    if (heading && heading.length <= 200)
-        return heading;
-    return null;
-}
-function extractProductUrl($, container, sourceUrl) {
-    const links = container.find('a[href]').toArray();
-    for (const link of links) {
-        const href = $(link).attr('href');
-        if (!href)
-            continue;
-        const text = $(link).text().replace(/\s+/g, ' ').trim().toLowerCase();
-        if (!text.includes('visit') && !text.includes('website'))
-            continue;
-        const resolved = resolveExternalUrl(href);
-        if (resolved && resolved !== sourceUrl)
-            return resolved;
-    }
-    for (const link of links) {
-        const href = $(link).attr('href');
-        const resolved = href ? resolveExternalUrl(href) : null;
-        if (resolved && !resolved.includes('betalist.com/startups/'))
-            return resolved;
-    }
-    return null;
-}
-function resolveExternalUrl(href) {
-    try {
-        const url = new URL(href, BETALIST_ORIGIN);
-        if (url.hostname === 'betalist.com' || url.hostname === 'www.betalist.com')
-            return null;
-        return url.toString();
-    }
-    catch {
-        return null;
-    }
-}
-function parseLaunchDateForRow($, container) {
-    const time = container.find('time[datetime]').first().attr('datetime');
-    const parsedTime = parseDate(time);
-    if (parsedTime)
-        return parsedTime;
-    const localDate = parseDate(container.find('time').first().text());
-    if (localDate)
-        return localDate;
-    const previous = container.prevAll('h1, h2, h3, h4, time, [datetime]').slice(0, 5).toArray();
-    for (const element of previous) {
-        const value = $(element).attr('datetime') ?? $(element).text();
-        const parsed = parseDate(value);
-        if (parsed)
-            return parsed;
-    }
-    const parentPrevious = container.parent().prevAll('h1, h2, h3, h4, time, [datetime]').slice(0, 5).toArray();
-    for (const element of parentPrevious) {
-        const value = $(element).attr('datetime') ?? $(element).text();
-        const parsed = parseDate(value);
-        if (parsed)
-            return parsed;
-    }
-    return null;
-}
-function parseDate(value) {
-    const trimmed = value?.replace(/\s+/g, ' ').trim();
-    if (!trimmed)
-        return null;
-    const normalized = trimmed
-        .replace(/^today$/i, new Date().toISOString())
-        .replace(/^yesterday$/i, new Date(Date.now() - 86_400_000).toISOString());
-    const parsed = new Date(normalized);
-    return Number.isNaN(parsed.getTime()) ? null : parsed;
-}
-function isWithinLookback(date, lookbackDays) {
-    const cutoff = new Date();
-    cutoff.setHours(0, 0, 0, 0);
-    cutoff.setDate(cutoff.getDate() - Math.max(1, lookbackDays));
-    return date >= cutoff;
-}
-function extractCreatorName($, container) {
-    const relAuthor = container.find('[rel="author"], [class*="creator"], [class*="maker"], a[href^="/makers/"]').first().text();
-    const normalized = relAuthor.replace(/\s+/g, ' ').trim();
-    return normalized || null;
-}
-function extractTagline($, container, productName) {
-    const selectors = ['[class*="tagline"]', '[class*="description"]', 'p'];
-    for (const selector of selectors) {
-        const text = container.find(selector).first().text().replace(/\s+/g, ' ').trim();
-        if (text && text !== productName && text.length <= 500)
-            return text;
-    }
-    return null;
-}
-function extractEmail(text) {
-    return text.match(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/)?.[0] ?? null;
-}
-function extractHandleFromHtml(html) {
-    const match = html.match(/(?:twitter\.com|x\.com)\/([A-Za-z0-9_]{1,20})/i);
-    return match ? `@${match[1]}` : null;
-}
-//# sourceMappingURL=scrape-betalist.js.map

package/dist/crm/scrape-landing.d.ts

@@ -1,24 +0,0 @@
-import type { Browser } from 'playwright';
-export type LandingScrape = {
-    htmlSignals: {
-        n_screenshots_html?: number;
-        has_video_html?: boolean;
-        has_docs_html?: boolean;
-        has_dashboard_html?: boolean;
-    };
-    screenshotPath: string | null;
-    isWebApp: boolean;
-    rawText: string;
-    error?: string;
-};
-export interface ScrapeLandingOptions {
-    productUrl: string;
-    userAgent: string;
-    logger: {
-        info(msg: string): void;
-        warn(msg: string): void;
-        error(msg: string): void;
-    };
-    browser: Browser;
-}
-export declare function scrapeLanding(opts: ScrapeLandingOptions): Promise<LandingScrape>;

package/dist/crm/scrape-landing.js

@@ -1,240 +0,0 @@
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-const VIDEO_HOST_RE = /(?:youtube\.com|youtu\.be|vimeo\.com|loom\.com|wistia)/i;
-const NOT_WEB_APP_RE = /\b(iOS only|App Store only|iPhone only|iPad only|Android only|Google Play only|hardware|physical product|chrome extension only|firefox addon only)\b/i;
-export async function scrapeLanding(opts) {
-    const url = normalizeHttpUrl(opts.productUrl);
-    if (!url) {
-        return {
-            htmlSignals: {},
-            screenshotPath: null,
-            isWebApp: true,
-            rawText: '',
-            error: 'invalid_product_url',
-        };
-    }
-    const robotsAllowed = await isAllowedByRobots(url, opts.userAgent, opts.logger);
-    if (!robotsAllowed) {
-        return {
-            htmlSignals: {},
-            screenshotPath: null,
-            isWebApp: true,
-            rawText: '',
-            error: 'robots_disallowed',
-        };
-    }
-    const htmlResult = await scrapeHtmlSignals(url, opts.userAgent, opts.logger);
-    // The orchestrator launches Chromium and applies Linux-only container flags.
-    try {
-        const screenshotPath = await captureLandingScreenshot(url, opts);
-        return { ...htmlResult, screenshotPath };
-    }
-    catch (error) {
-        opts.logger.warn(`[crm-landing] Screenshot failed for ${url}: ${error.message}`);
-        return {
-            ...htmlResult,
-            screenshotPath: null,
-            error: `screenshot_failed: ${error.message}`,
-        };
-    }
-}
-async function isAllowedByRobots(url, userAgent, logger) {
-    let robotsUrl;
-    try {
-        robotsUrl = new URL('/robots.txt', url).toString();
-    }
-    catch {
-        return true;
-    }
-    try {
-        const response = await fetch(robotsUrl, {
-            headers: {
-                Accept: 'text/plain,text/*,*/*',
-                'User-Agent': userAgent,
-            },
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!response.ok)
-            return true;
-        const text = await response.text();
-        return !robotsDisallowsAll(text);
-    }
-    catch (error) {
-        logger.warn(`[crm-landing] robots.txt fetch failed open for ${robotsUrl}: ${error.message}`);
-        return true;
-    }
-}
-function robotsDisallowsAll(text) {
-    const groups = [];
-    let current = null;
-    for (const rawLine of text.split(/\r?\n/)) {
-        const line = rawLine.replace(/#.*/, '').trim();
-        if (!line)
-            continue;
-        const colon = line.indexOf(':');
-        if (colon === -1)
-            continue;
-        const key = line.slice(0, colon).trim().toLowerCase();
-        const value = line.slice(colon + 1).trim();
-        if (key === 'user-agent') {
-            if (!current || current.disallows.length > 0) {
-                current = { agents: [], disallows: [] };
-                groups.push(current);
-            }
-            current.agents.push(value.toLowerCase());
-        }
-        else if (key === 'disallow' && current) {
-            current.disallows.push(value);
-        }
-    }
-    const autokap = groups.find((group) => group.agents.includes('autokap-crawler'));
-    const wildcard = groups.find((group) => group.agents.includes('*'));
-    const group = autokap ?? wildcard;
-    return group?.disallows.some((rule) => rule.trim() === '/') ?? false;
-}
-async function scrapeHtmlSignals(url, userAgent, logger) {
-    try {
-        const response = await fetch(url, {
-            headers: {
-                Accept: 'text/html',
-                'User-Agent': userAgent,
-            },
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!response.ok) {
-            logger.warn(`[crm-landing] HTML fetch returned HTTP ${response.status} for ${url}`);
-            return { htmlSignals: {}, isWebApp: true, rawText: '' };
-        }
-        const html = await response.text();
-        const cheerio = await loadCheerio();
-        const $ = cheerio.load(html);
-        $('script, style, noscript, svg').remove();
-        const rawText = $('body').text().replace(/\s+/g, ' ').trim().slice(0, 8000);
-        const htmlSignals = {
-            n_screenshots_html: countContentImages($),
-            has_video_html: hasVideo($),
-            has_docs_html: hasDocs($),
-            has_dashboard_html: hasDashboard(rawText),
-        };
-        return {
-            htmlSignals,
-            isWebApp: detectWebApp($, rawText),
-            rawText,
-        };
-    }
-    catch (error) {
-        logger.warn(`[crm-landing] HTML fetch failed for ${url}: ${error.message}`);
-        return { htmlSignals: {}, isWebApp: true, rawText: '' };
-    }
-}
-async function loadCheerio() {
-    const importer = new Function('specifier', 'return import(specifier)');
-    return importer('cheerio');
-}
-function countContentImages($) {
-    let count = 0;
-    $('img[src]').each((_, img) => {
-        if (count >= 50)
-            return;
-        const node = $(img);
-        const src = node.attr('src') ?? '';
-        const className = node.attr('class') ?? '';
-        const width = parseDimension(node.attr('width'));
-        const height = parseDimension(node.attr('height'));
-        let pathname = '';
-        try {
-            pathname = new URL(src, 'https://example.test/').pathname;
-        }
-        catch {
-            pathname = src;
-        }
-        if (/logo|icon/i.test(className))
-            return;
-        if (/logo|favicon/i.test(pathname))
-            return;
-        if ((width !== null && width <= 64) || (height !== null && height <= 64))
-            return;
-        count += 1;
-    });
-    return count;
-}
-function parseDimension(value) {
-    if (!value)
-        return null;
-    const parsed = Number.parseInt(value, 10);
-    return Number.isFinite(parsed) ? parsed : null;
-}
-function hasVideo($) {
-    if ($('video').length > 0)
-        return true;
-    return $('iframe[src]').toArray().some((iframe) => VIDEO_HOST_RE.test($(iframe).attr('src') ?? ''));
-}
-function hasDocs($) {
-    return $('a[href]').toArray().some((anchor) => {
-        const href = ($(anchor).attr('href') ?? '').toLowerCase();
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim().toLowerCase();
-        return href.includes('/docs') || href.includes('/documentation') || text === 'docs' || text === 'documentation';
-    });
-}
-function hasDashboard(text) {
-    if (/\bdashboard\b/i.test(text))
-        return true;
-    if (!/\bapp\b/i.test(text))
-        return false;
-    return /\b(sign in|log in|login|signup)\b/i.test(text);
-}
-function detectWebApp($, rawText) {
-    if (NOT_WEB_APP_RE.test(rawText))
-        return false;
-    const ctaLinks = $('a[href]').toArray().filter((anchor) => {
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
-        const href = $(anchor).attr('href') ?? '';
-        return /app store|google play/i.test(`${text} ${href}`);
-    });
-    const primaryLinks = $('a[href]').toArray().filter((anchor) => {
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
-        if (!text)
-            return false;
-        return /download|get|install|app store|google play/i.test(text);
-    });
-    return !(primaryLinks.length > 0 && primaryLinks.length === ctaLinks.length);
-}
-async function captureLandingScreenshot(url, opts) {
-    const context = await opts.browser.newContext({
-        userAgent: opts.userAgent,
-        viewport: { width: 1280, height: 800 },
-    });
-    try {
-        const page = await context.newPage();
-        try {
-            await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 });
-        }
-        catch (error) {
-            const err = error;
-            opts.logger.warn(`[crm-landing] networkidle goto failed for ${url}, retrying load: ${err.message}`);
-            await page.goto(url, { waitUntil: 'load', timeout: 30_000 });
-        }
-        const dir = path.join(os.tmpdir(), 'autokap-crm');
-        await fs.mkdir(dir, { recursive: true });
-        const slug = new URL(url).hostname.replace(/[^a-z0-9.-]+/gi, '-').replace(/^-+|-+$/g, '') || 'landing';
-        const screenshotPath = path.join(dir, `landing-${slug}-${Date.now()}.png`);
-        await page.screenshot({ path: screenshotPath, fullPage: true, type: 'png' });
-        return screenshotPath;
-    }
-    finally {
-        await context.close();
-    }
-}
-function normalizeHttpUrl(value) {
-    try {
-        const url = new URL(value);
-        if (url.protocol !== 'http:' && url.protocol !== 'https:')
-            return null;
-        return url.toString();
-    }
-    catch {
-        return null;
-    }
-}
-//# sourceMappingURL=scrape-landing.js.map

package/dist/crm/storage-upload.d.ts

@@ -1,14 +0,0 @@
-export interface UploadLandingShotOptions {
-    runId: string;
-    sourceUrl: string;
-    pngBuffer: Buffer;
-    apiBaseUrl: string;
-    runToken: string;
-}
-export declare class LandingShotEndpointMissingError extends Error {
-    constructor();
-}
-export declare function uploadLandingShot(opts: UploadLandingShotOptions): Promise<{
-    signedUrl: string;
-    expiresInSec: number;
-}>;

package/dist/crm/storage-upload.js

@@ -1,40 +0,0 @@
-// TODO(parent): backend endpoint POST /api/cli/crm/runs/[runId]/landing-shot is referenced here but may not exist yet — see comment.
-import { logger } from '../logger.js';
-export class LandingShotEndpointMissingError extends Error {
-    constructor() {
-        super('LANDING_SHOT_ENDPOINT_MISSING');
-        this.name = 'LandingShotEndpointMissingError';
-    }
-}
-export async function uploadLandingShot(opts) {
-    const form = new FormData();
-    form.set('source_url', opts.sourceUrl);
-    const bytes = new Uint8Array(opts.pngBuffer.byteLength);
-    bytes.set(opts.pngBuffer);
-    form.set('file', new Blob([bytes.buffer], { type: 'image/png' }), 'landing.png');
-    const response = await fetch(`${opts.apiBaseUrl.replace(/\/+$/, '')}/api/cli/crm/runs/${opts.runId}/landing-shot`, {
-        method: 'POST',
-        headers: {
-            Authorization: `Bearer ${opts.runToken}`,
-        },
-        body: form,
-        signal: AbortSignal.timeout(30_000),
-    });
-    if (response.status === 404) {
-        logger.warn('[crm-upload] Landing-shot endpoint missing; continuing without vision');
-        throw new LandingShotEndpointMissingError();
-    }
-    if (!response.ok) {
-        const body = await response.text().catch(() => response.statusText);
-        throw new Error(`landing-shot upload failed: HTTP ${response.status} ${body.slice(0, 300)}`);
-    }
-    const json = await response.json().catch(() => null);
-    if (!json || typeof json.signedUrl !== 'string' || typeof json.expiresInSec !== 'number') {
-        throw new Error('landing-shot upload returned invalid JSON');
-    }
-    return {
-        signedUrl: json.signedUrl,
-        expiresInSec: json.expiresInSec,
-    };
-}
-//# sourceMappingURL=storage-upload.js.map