autokap 1.4.3 → 1.5.1

package/dist/crm/scrape-landing.js

@@ -1,240 +0,0 @@
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-const VIDEO_HOST_RE = /(?:youtube\.com|youtu\.be|vimeo\.com|loom\.com|wistia)/i;
-const NOT_WEB_APP_RE = /\b(iOS only|App Store only|iPhone only|iPad only|Android only|Google Play only|hardware|physical product|chrome extension only|firefox addon only)\b/i;
-export async function scrapeLanding(opts) {
-    const url = normalizeHttpUrl(opts.productUrl);
-    if (!url) {
-        return {
-            htmlSignals: {},
-            screenshotPath: null,
-            isWebApp: true,
-            rawText: '',
-            error: 'invalid_product_url',
-        };
-    }
-    const robotsAllowed = await isAllowedByRobots(url, opts.userAgent, opts.logger);
-    if (!robotsAllowed) {
-        return {
-            htmlSignals: {},
-            screenshotPath: null,
-            isWebApp: true,
-            rawText: '',
-            error: 'robots_disallowed',
-        };
-    }
-    const htmlResult = await scrapeHtmlSignals(url, opts.userAgent, opts.logger);
-    // The orchestrator launches Chromium and applies Linux-only container flags.
-    try {
-        const screenshotPath = await captureLandingScreenshot(url, opts);
-        return { ...htmlResult, screenshotPath };
-    }
-    catch (error) {
-        opts.logger.warn(`[crm-landing] Screenshot failed for ${url}: ${error.message}`);
-        return {
-            ...htmlResult,
-            screenshotPath: null,
-            error: `screenshot_failed: ${error.message}`,
-        };
-    }
-}
-async function isAllowedByRobots(url, userAgent, logger) {
-    let robotsUrl;
-    try {
-        robotsUrl = new URL('/robots.txt', url).toString();
-    }
-    catch {
-        return true;
-    }
-    try {
-        const response = await fetch(robotsUrl, {
-            headers: {
-                Accept: 'text/plain,text/*,*/*',
-                'User-Agent': userAgent,
-            },
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!response.ok)
-            return true;
-        const text = await response.text();
-        return !robotsDisallowsAll(text);
-    }
-    catch (error) {
-        logger.warn(`[crm-landing] robots.txt fetch failed open for ${robotsUrl}: ${error.message}`);
-        return true;
-    }
-}
-function robotsDisallowsAll(text) {
-    const groups = [];
-    let current = null;
-    for (const rawLine of text.split(/\r?\n/)) {
-        const line = rawLine.replace(/#.*/, '').trim();
-        if (!line)
-            continue;
-        const colon = line.indexOf(':');
-        if (colon === -1)
-            continue;
-        const key = line.slice(0, colon).trim().toLowerCase();
-        const value = line.slice(colon + 1).trim();
-        if (key === 'user-agent') {
-            if (!current || current.disallows.length > 0) {
-                current = { agents: [], disallows: [] };
-                groups.push(current);
-            }
-            current.agents.push(value.toLowerCase());
-        }
-        else if (key === 'disallow' && current) {
-            current.disallows.push(value);
-        }
-    }
-    const autokap = groups.find((group) => group.agents.includes('autokap-crawler'));
-    const wildcard = groups.find((group) => group.agents.includes('*'));
-    const group = autokap ?? wildcard;
-    return group?.disallows.some((rule) => rule.trim() === '/') ?? false;
-}
-async function scrapeHtmlSignals(url, userAgent, logger) {
-    try {
-        const response = await fetch(url, {
-            headers: {
-                Accept: 'text/html',
-                'User-Agent': userAgent,
-            },
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!response.ok) {
-            logger.warn(`[crm-landing] HTML fetch returned HTTP ${response.status} for ${url}`);
-            return { htmlSignals: {}, isWebApp: true, rawText: '' };
-        }
-        const html = await response.text();
-        const cheerio = await loadCheerio();
-        const $ = cheerio.load(html);
-        $('script, style, noscript, svg').remove();
-        const rawText = $('body').text().replace(/\s+/g, ' ').trim().slice(0, 8000);
-        const htmlSignals = {
-            n_screenshots_html: countContentImages($),
-            has_video_html: hasVideo($),
-            has_docs_html: hasDocs($),
-            has_dashboard_html: hasDashboard(rawText),
-        };
-        return {
-            htmlSignals,
-            isWebApp: detectWebApp($, rawText),
-            rawText,
-        };
-    }
-    catch (error) {
-        logger.warn(`[crm-landing] HTML fetch failed for ${url}: ${error.message}`);
-        return { htmlSignals: {}, isWebApp: true, rawText: '' };
-    }
-}
-async function loadCheerio() {
-    const importer = new Function('specifier', 'return import(specifier)');
-    return importer('cheerio');
-}
-function countContentImages($) {
-    let count = 0;
-    $('img[src]').each((_, img) => {
-        if (count >= 50)
-            return;
-        const node = $(img);
-        const src = node.attr('src') ?? '';
-        const className = node.attr('class') ?? '';
-        const width = parseDimension(node.attr('width'));
-        const height = parseDimension(node.attr('height'));
-        let pathname = '';
-        try {
-            pathname = new URL(src, 'https://example.test/').pathname;
-        }
-        catch {
-            pathname = src;
-        }
-        if (/logo|icon/i.test(className))
-            return;
-        if (/logo|favicon/i.test(pathname))
-            return;
-        if ((width !== null && width <= 64) || (height !== null && height <= 64))
-            return;
-        count += 1;
-    });
-    return count;
-}
-function parseDimension(value) {
-    if (!value)
-        return null;
-    const parsed = Number.parseInt(value, 10);
-    return Number.isFinite(parsed) ? parsed : null;
-}
-function hasVideo($) {
-    if ($('video').length > 0)
-        return true;
-    return $('iframe[src]').toArray().some((iframe) => VIDEO_HOST_RE.test($(iframe).attr('src') ?? ''));
-}
-function hasDocs($) {
-    return $('a[href]').toArray().some((anchor) => {
-        const href = ($(anchor).attr('href') ?? '').toLowerCase();
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim().toLowerCase();
-        return href.includes('/docs') || href.includes('/documentation') || text === 'docs' || text === 'documentation';
-    });
-}
-function hasDashboard(text) {
-    if (/\bdashboard\b/i.test(text))
-        return true;
-    if (!/\bapp\b/i.test(text))
-        return false;
-    return /\b(sign in|log in|login|signup)\b/i.test(text);
-}
-function detectWebApp($, rawText) {
-    if (NOT_WEB_APP_RE.test(rawText))
-        return false;
-    const ctaLinks = $('a[href]').toArray().filter((anchor) => {
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
-        const href = $(anchor).attr('href') ?? '';
-        return /app store|google play/i.test(`${text} ${href}`);
-    });
-    const primaryLinks = $('a[href]').toArray().filter((anchor) => {
-        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
-        if (!text)
-            return false;
-        return /download|get|install|app store|google play/i.test(text);
-    });
-    return !(primaryLinks.length > 0 && primaryLinks.length === ctaLinks.length);
-}
-async function captureLandingScreenshot(url, opts) {
-    const context = await opts.browser.newContext({
-        userAgent: opts.userAgent,
-        viewport: { width: 1280, height: 800 },
-    });
-    try {
-        const page = await context.newPage();
-        try {
-            await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 });
-        }
-        catch (error) {
-            const err = error;
-            opts.logger.warn(`[crm-landing] networkidle goto failed for ${url}, retrying load: ${err.message}`);
-            await page.goto(url, { waitUntil: 'load', timeout: 30_000 });
-        }
-        const dir = path.join(os.tmpdir(), 'autokap-crm');
-        await fs.mkdir(dir, { recursive: true });
-        const slug = new URL(url).hostname.replace(/[^a-z0-9.-]+/gi, '-').replace(/^-+|-+$/g, '') || 'landing';
-        const screenshotPath = path.join(dir, `landing-${slug}-${Date.now()}.png`);
-        await page.screenshot({ path: screenshotPath, fullPage: true, type: 'png' });
-        return screenshotPath;
-    }
-    finally {
-        await context.close();
-    }
-}
-function normalizeHttpUrl(value) {
-    try {
-        const url = new URL(value);
-        if (url.protocol !== 'http:' && url.protocol !== 'https:')
-            return null;
-        return url.toString();
-    }
-    catch {
-        return null;
-    }
-}
-//# sourceMappingURL=scrape-landing.js.map

package/dist/crm/storage-upload.d.ts

@@ -1,14 +0,0 @@
-export interface UploadLandingShotOptions {
-    runId: string;
-    sourceUrl: string;
-    pngBuffer: Buffer;
-    apiBaseUrl: string;
-    runToken: string;
-}
-export declare class LandingShotEndpointMissingError extends Error {
-    constructor();
-}
-export declare function uploadLandingShot(opts: UploadLandingShotOptions): Promise<{
-    signedUrl: string;
-    expiresInSec: number;
-}>;

package/dist/crm/storage-upload.js

@@ -1,40 +0,0 @@
-// TODO(parent): backend endpoint POST /api/cli/crm/runs/[runId]/landing-shot is referenced here but may not exist yet — see comment.
-import { logger } from '../logger.js';
-export class LandingShotEndpointMissingError extends Error {
-    constructor() {
-        super('LANDING_SHOT_ENDPOINT_MISSING');
-        this.name = 'LandingShotEndpointMissingError';
-    }
-}
-export async function uploadLandingShot(opts) {
-    const form = new FormData();
-    form.set('source_url', opts.sourceUrl);
-    const bytes = new Uint8Array(opts.pngBuffer.byteLength);
-    bytes.set(opts.pngBuffer);
-    form.set('file', new Blob([bytes.buffer], { type: 'image/png' }), 'landing.png');
-    const response = await fetch(`${opts.apiBaseUrl.replace(/\/+$/, '')}/api/cli/crm/runs/${opts.runId}/landing-shot`, {
-        method: 'POST',
-        headers: {
-            Authorization: `Bearer ${opts.runToken}`,
-        },
-        body: form,
-        signal: AbortSignal.timeout(30_000),
-    });
-    if (response.status === 404) {
-        logger.warn('[crm-upload] Landing-shot endpoint missing; continuing without vision');
-        throw new LandingShotEndpointMissingError();
-    }
-    if (!response.ok) {
-        const body = await response.text().catch(() => response.statusText);
-        throw new Error(`landing-shot upload failed: HTTP ${response.status} ${body.slice(0, 300)}`);
-    }
-    const json = await response.json().catch(() => null);
-    if (!json || typeof json.signedUrl !== 'string' || typeof json.expiresInSec !== 'number') {
-        throw new Error('landing-shot upload returned invalid JSON');
-    }
-    return {
-        signedUrl: json.signedUrl,
-        expiresInSec: json.expiresInSec,
-    };
-}
-//# sourceMappingURL=storage-upload.js.map