autokap 1.4.0 → 1.4.2

package/dist/crm/scrape-landing.js

@@ -0,0 +1,240 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+const VIDEO_HOST_RE = /(?:youtube\.com|youtu\.be|vimeo\.com|loom\.com|wistia)/i;
+const NOT_WEB_APP_RE = /\b(iOS only|App Store only|iPhone only|iPad only|Android only|Google Play only|hardware|physical product|chrome extension only|firefox addon only)\b/i;
+export async function scrapeLanding(opts) {
+    const url = normalizeHttpUrl(opts.productUrl);
+    if (!url) {
+        return {
+            htmlSignals: {},
+            screenshotPath: null,
+            isWebApp: true,
+            rawText: '',
+            error: 'invalid_product_url',
+        };
+    }
+    const robotsAllowed = await isAllowedByRobots(url, opts.userAgent, opts.logger);
+    if (!robotsAllowed) {
+        return {
+            htmlSignals: {},
+            screenshotPath: null,
+            isWebApp: true,
+            rawText: '',
+            error: 'robots_disallowed',
+        };
+    }
+    const htmlResult = await scrapeHtmlSignals(url, opts.userAgent, opts.logger);
+    // The orchestrator launches Chromium and applies Linux-only container flags.
+    try {
+        const screenshotPath = await captureLandingScreenshot(url, opts);
+        return { ...htmlResult, screenshotPath };
+    }
+    catch (error) {
+        opts.logger.warn(`[crm-landing] Screenshot failed for ${url}: ${error.message}`);
+        return {
+            ...htmlResult,
+            screenshotPath: null,
+            error: `screenshot_failed: ${error.message}`,
+        };
+    }
+}
+async function isAllowedByRobots(url, userAgent, logger) {
+    let robotsUrl;
+    try {
+        robotsUrl = new URL('/robots.txt', url).toString();
+    }
+    catch {
+        return true;
+    }
+    try {
+        const response = await fetch(robotsUrl, {
+            headers: {
+                Accept: 'text/plain,text/*,*/*',
+                'User-Agent': userAgent,
+            },
+            signal: AbortSignal.timeout(15_000),
+        });
+        if (!response.ok)
+            return true;
+        const text = await response.text();
+        return !robotsDisallowsAll(text);
+    }
+    catch (error) {
+        logger.warn(`[crm-landing] robots.txt fetch failed open for ${robotsUrl}: ${error.message}`);
+        return true;
+    }
+}
+function robotsDisallowsAll(text) {
+    const groups = [];
+    let current = null;
+    for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*/, '').trim();
+        if (!line)
+            continue;
+        const colon = line.indexOf(':');
+        if (colon === -1)
+            continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === 'user-agent') {
+            if (!current || current.disallows.length > 0) {
+                current = { agents: [], disallows: [] };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+        }
+        else if (key === 'disallow' && current) {
+            current.disallows.push(value);
+        }
+    }
+    const autokap = groups.find((group) => group.agents.includes('autokap-crawler'));
+    const wildcard = groups.find((group) => group.agents.includes('*'));
+    const group = autokap ?? wildcard;
+    return group?.disallows.some((rule) => rule.trim() === '/') ?? false;
+}
+async function scrapeHtmlSignals(url, userAgent, logger) {
+    try {
+        const response = await fetch(url, {
+            headers: {
+                Accept: 'text/html',
+                'User-Agent': userAgent,
+            },
+            signal: AbortSignal.timeout(15_000),
+        });
+        if (!response.ok) {
+            logger.warn(`[crm-landing] HTML fetch returned HTTP ${response.status} for ${url}`);
+            return { htmlSignals: {}, isWebApp: true, rawText: '' };
+        }
+        const html = await response.text();
+        const cheerio = await loadCheerio();
+        const $ = cheerio.load(html);
+        $('script, style, noscript, svg').remove();
+        const rawText = $('body').text().replace(/\s+/g, ' ').trim().slice(0, 8000);
+        const htmlSignals = {
+            n_screenshots_html: countContentImages($),
+            has_video_html: hasVideo($),
+            has_docs_html: hasDocs($),
+            has_dashboard_html: hasDashboard(rawText),
+        };
+        return {
+            htmlSignals,
+            isWebApp: detectWebApp($, rawText),
+            rawText,
+        };
+    }
+    catch (error) {
+        logger.warn(`[crm-landing] HTML fetch failed for ${url}: ${error.message}`);
+        return { htmlSignals: {}, isWebApp: true, rawText: '' };
+    }
+}
+async function loadCheerio() {
+    const importer = new Function('specifier', 'return import(specifier)');
+    return importer('cheerio');
+}
+function countContentImages($) {
+    let count = 0;
+    $('img[src]').each((_, img) => {
+        if (count >= 50)
+            return;
+        const node = $(img);
+        const src = node.attr('src') ?? '';
+        const className = node.attr('class') ?? '';
+        const width = parseDimension(node.attr('width'));
+        const height = parseDimension(node.attr('height'));
+        let pathname = '';
+        try {
+            pathname = new URL(src, 'https://example.test/').pathname;
+        }
+        catch {
+            pathname = src;
+        }
+        if (/logo|icon/i.test(className))
+            return;
+        if (/logo|favicon/i.test(pathname))
+            return;
+        if ((width !== null && width <= 64) || (height !== null && height <= 64))
+            return;
+        count += 1;
+    });
+    return count;
+}
+function parseDimension(value) {
+    if (!value)
+        return null;
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function hasVideo($) {
+    if ($('video').length > 0)
+        return true;
+    return $('iframe[src]').toArray().some((iframe) => VIDEO_HOST_RE.test($(iframe).attr('src') ?? ''));
+}
+function hasDocs($) {
+    return $('a[href]').toArray().some((anchor) => {
+        const href = ($(anchor).attr('href') ?? '').toLowerCase();
+        const text = $(anchor).text().replace(/\s+/g, ' ').trim().toLowerCase();
+        return href.includes('/docs') || href.includes('/documentation') || text === 'docs' || text === 'documentation';
+    });
+}
+function hasDashboard(text) {
+    if (/\bdashboard\b/i.test(text))
+        return true;
+    if (!/\bapp\b/i.test(text))
+        return false;
+    return /\b(sign in|log in|login|signup)\b/i.test(text);
+}
+function detectWebApp($, rawText) {
+    if (NOT_WEB_APP_RE.test(rawText))
+        return false;
+    const ctaLinks = $('a[href]').toArray().filter((anchor) => {
+        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
+        const href = $(anchor).attr('href') ?? '';
+        return /app store|google play/i.test(`${text} ${href}`);
+    });
+    const primaryLinks = $('a[href]').toArray().filter((anchor) => {
+        const text = $(anchor).text().replace(/\s+/g, ' ').trim();
+        if (!text)
+            return false;
+        return /download|get|install|app store|google play/i.test(text);
+    });
+    return !(primaryLinks.length > 0 && primaryLinks.length === ctaLinks.length);
+}
+async function captureLandingScreenshot(url, opts) {
+    const context = await opts.browser.newContext({
+        userAgent: opts.userAgent,
+        viewport: { width: 1280, height: 800 },
+    });
+    try {
+        const page = await context.newPage();
+        try {
+            await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 });
+        }
+        catch (error) {
+            const err = error;
+            opts.logger.warn(`[crm-landing] networkidle goto failed for ${url}, retrying load: ${err.message}`);
+            await page.goto(url, { waitUntil: 'load', timeout: 30_000 });
+        }
+        const dir = path.join(os.tmpdir(), 'autokap-crm');
+        await fs.mkdir(dir, { recursive: true });
+        const slug = new URL(url).hostname.replace(/[^a-z0-9.-]+/gi, '-').replace(/^-+|-+$/g, '') || 'landing';
+        const screenshotPath = path.join(dir, `landing-${slug}-${Date.now()}.png`);
+        await page.screenshot({ path: screenshotPath, fullPage: true, type: 'png' });
+        return screenshotPath;
+    }
+    finally {
+        await context.close();
+    }
+}
+function normalizeHttpUrl(value) {
+    try {
+        const url = new URL(value);
+        if (url.protocol !== 'http:' && url.protocol !== 'https:')
+            return null;
+        return url.toString();
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=scrape-landing.js.map

package/dist/crm/storage-upload.d.ts

@@ -0,0 +1,14 @@
+export interface UploadLandingShotOptions {
+    runId: string;
+    sourceUrl: string;
+    pngBuffer: Buffer;
+    apiBaseUrl: string;
+    runToken: string;
+}
+export declare class LandingShotEndpointMissingError extends Error {
+    constructor();
+}
+export declare function uploadLandingShot(opts: UploadLandingShotOptions): Promise<{
+    signedUrl: string;
+    expiresInSec: number;
+}>;

package/dist/crm/storage-upload.js

@@ -0,0 +1,40 @@
+// TODO(parent): backend endpoint POST /api/cli/crm/runs/[runId]/landing-shot is referenced here but may not exist yet — see comment.
+import { logger } from '../logger.js';
+export class LandingShotEndpointMissingError extends Error {
+    constructor() {
+        super('LANDING_SHOT_ENDPOINT_MISSING');
+        this.name = 'LandingShotEndpointMissingError';
+    }
+}
+export async function uploadLandingShot(opts) {
+    const form = new FormData();
+    form.set('source_url', opts.sourceUrl);
+    const bytes = new Uint8Array(opts.pngBuffer.byteLength);
+    bytes.set(opts.pngBuffer);
+    form.set('file', new Blob([bytes.buffer], { type: 'image/png' }), 'landing.png');
+    const response = await fetch(`${opts.apiBaseUrl.replace(/\/+$/, '')}/api/cli/crm/runs/${opts.runId}/landing-shot`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${opts.runToken}`,
+        },
+        body: form,
+        signal: AbortSignal.timeout(30_000),
+    });
+    if (response.status === 404) {
+        logger.warn('[crm-upload] Landing-shot endpoint missing; continuing without vision');
+        throw new LandingShotEndpointMissingError();
+    }
+    if (!response.ok) {
+        const body = await response.text().catch(() => response.statusText);
+        throw new Error(`landing-shot upload failed: HTTP ${response.status} ${body.slice(0, 300)}`);
+    }
+    const json = await response.json().catch(() => null);
+    if (!json || typeof json.signedUrl !== 'string' || typeof json.expiresInSec !== 'number') {
+        throw new Error('landing-shot upload returned invalid JSON');
+    }
+    return {
+        signedUrl: json.signedUrl,
+        expiresInSec: json.expiresInSec,
+    };
+}
+//# sourceMappingURL=storage-upload.js.map

package/dist/mockup.d.ts

@@ -95,6 +95,13 @@ interface OrientationConfigData {
     };
     /** Which coded browser chrome style to render. Defaults to 'chrome'. */
     browserStyle?: 'chrome' | 'safari';
+    adminShowStatusBar?: boolean;
+    adminForcedSafeAreaColors?: {
+        top?: string;
+        bottom?: string;
+        left?: string;
+        right?: string;
+    };
 }
 export type MockupOrientation = 'portrait' | 'landscape';
 export interface MockupOptions {

package/dist/mockup.js

@@ -116,6 +116,8 @@ function resolveOrientationConfig(config, requestedOrientation) {
             needsRotation: false,
             disableOverlays: false,
             frameBehindContent: orientationData.frameBehindContent ?? false,
+            adminShowStatusBar: orientationData.adminShowStatusBar,
+            adminForcedSafeAreaColors: orientationData.adminForcedSafeAreaColors,
         };
     }
     // Legacy: single config, may need auto-rotation
@@ -138,6 +140,8 @@ function resolveOrientationConfig(config, requestedOrientation) {
         needsRotation,
         disableOverlays,
         frameBehindContent: config.frameBehindContent ?? false,
+        adminShowStatusBar: config.adminShowStatusBar,
+        adminForcedSafeAreaColors: config.adminForcedSafeAreaColors,
     };
 }
 async function loadDeviceConfigs() {
@@ -571,6 +575,12 @@ export async function applyDeviceFrame(screenshot, deviceId, options) {
     // Resizing (especially large downscales with fit:'fill') averages edge pixels,
     // producing grayish artifacts that make safe area fills look darker than intended.
     const colors = await sampleEdgeColors(screenshot);
+    // For phone/tablet, clamp the top safe area fill (where the iOS status bar
+    // text/icons render) to pure white or pure black. Real iOS never tints the
+    // status bar, so a sampled blue/green/red top would look fake.
+    // This runs BEFORE the user/admin override checks below so explicit colours
+    // (opts.safeAreaTopColor or adminForcedColors.top) still win.
+    colors.topColor = clampMobileStatusBarFill(colors.topColor, config.category);
     // Resize screenshot to physical content dimensions (contentW*os × contentH*os).
     // The mockup HTML container is sized at logical dimensions (contentW × contentH)
     // with the <img> at width:100%;height:100%. Playwright renders the mockup at
@@ -624,12 +634,32 @@ export async function applyDeviceFrame(screenshot, deviceId, options) {
         : undefined;
     // Safe area toggles: when hidden, make fills transparent instead of removing them.
     // This keeps the content area (screenshot placement) at the viewport-matching size.
+    // Admin-forced colors take precedence over user overrides and auto-sampling.
+    const adminForcedColors = resolved.adminForcedSafeAreaColors ?? {};
     const safeAreaFillColors = {
-        top: isLaptop ? '#000000' : (opts.showSafeAreaTop ? colors.topColor : 'transparent'),
-        bottom: opts.showSafeAreaBottom ? colors.bottomColor : 'transparent',
-        left: opts.showSafeAreaLeft ? colors.leftColor : 'transparent',
-        right: opts.showSafeAreaRight ? colors.rightColor : 'transparent',
+        top: adminForcedColors.top ?? (opts.showSafeAreaTop ? colors.topColor : 'transparent'),
+        bottom: adminForcedColors.bottom ?? (opts.showSafeAreaBottom ? colors.bottomColor : 'transparent'),
+        left: adminForcedColors.left ?? (opts.showSafeAreaLeft ? colors.leftColor : 'transparent'),
+        right: adminForcedColors.right ?? (opts.showSafeAreaRight ? colors.rightColor : 'transparent'),
     };
+    // Status bar text contrast: derive *only* from the actually visible
+    // background behind the status bar text — admin-forced fill if set,
+    // otherwise the sampled (and possibly mobile-clamped) top color.
+    //
+    // We deliberately ignore `opts.colorScheme` here: that option represents the
+    // captured *page* theme (light/dark mode of the app being captured), not the
+    // colour of the safe-area fill. Honoring it would produce black text on a
+    // black fill (or white on white) when the page theme and the fill colour
+    // disagree — exactly the bug we're trying to avoid.
+    //
+    // `opts.statusBar.colorScheme` is still respected as a per-status-bar manual
+    // override, since that's a deliberate choice scoped to the bar itself.
+    const statusBarBg = adminForcedColors.top ?? colors.topColor;
+    const explicitStatusBarScheme = opts.statusBar?.colorScheme;
+    const statusBarColorScheme = isLaptop
+        ? 'dark'
+        : (explicitStatusBarScheme
+            ?? (isDarkBackground(statusBarBg) ? 'dark' : 'light'));
     const wbw = browserWindowBorder?.width ?? 0;
     const renderW = Math.round(geo.frameWidth);
     const renderH = Math.round(geo.frameHeight);
@@ -711,12 +741,13 @@ export async function applyDeviceFrame(screenshot, deviceId, options) {
         // Status bar uses satori (CSS flexbox engine) for pixel-perfect parity
         // with the client HTML rendering. Generated at logical dimensions, then
         // rasterized at physical pixel width.
-        const showStatusBar = !isBrowserDevice && (isLaptop || opts.showStatusBar);
+        const adminShowSB = resolved.adminShowStatusBar;
+        const showStatusBar = !isBrowserDevice && (adminShowSB !== undefined ? adminShowSB : opts.showStatusBar);
         if (showStatusBar && resolved.statusBar && resolved.safeArea.top > 0) {
             const sbW_logical = Math.round(sr.width);
             const sbH_logical = Math.round(resolved.safeArea.top * scale);
             const sbPng = await renderStatusBarBuffer({
-                config: { ...opts.statusBar, colorScheme: autoColorScheme },
+                config: { ...opts.statusBar, colorScheme: statusBarColorScheme },
                 width: sbW_logical,
                 height: sbH_logical,
                 scale,
@@ -858,6 +889,21 @@ function snapEdgeColor(rgb) {
         return 'rgb(0,0,0)';
     return rgb;
 }
+/**
+ * Real iPhones and iPads only show their status bar text/icons in pure white or
+ * pure black — never tinted. The auto-sampled top edge color often produces
+ * colored fills (e.g. blue from a hero banner) that make the safe area look
+ * unrealistic. Clamp the top color to pure white or pure black based on
+ * luminance for phone and tablet devices so the status bar reads cleanly.
+ *
+ * Bottom/left/right are left untouched (the bottom home indicator and side
+ * fills can legitimately match the app's edge tone).
+ */
+function clampMobileStatusBarFill(rgb, category) {
+    if (category !== 'phone' && category !== 'tablet')
+        return rgb;
+    return parseLuminance(rgb) >= 128 ? 'rgb(255,255,255)' : 'rgb(0,0,0)';
+}
 function channelsToRgb(channels) {
     const r = Math.round(channels[0].mean);
     const g = Math.round(channels[1].mean);

package/dist/types.d.ts

@@ -583,7 +583,7 @@ export interface ClipOptions {
 /** Usage metadata from a single OpenRouter API call */
 export interface StepUsage {
     stepNumber: number;
-    stepType: 'agent_iteration' | 'verification' | 'element_capture' | 'video_planning' | 'video_variant_classification' | 'video_step_verification' | 'video_step_fix' | 'assistant_chat' | 'studio_creation' | 'studio_iteration' | 'studio_capture_suggestion' | 'mock_data_generation' | 'page_identity_classification' | 'capture_verification' | 'alt_text_generation' | 'healer_invocation' | 'cron_feedback_classification' | 'tts_generation';
+    stepType: 'agent_iteration' | 'verification' | 'element_capture' | 'video_planning' | 'video_variant_classification' | 'video_step_verification' | 'video_step_fix' | 'assistant_chat' | 'studio_creation' | 'studio_iteration' | 'studio_capture_suggestion' | 'mock_data_generation' | 'page_identity_classification' | 'capture_verification' | 'alt_text_generation' | 'healer_invocation' | 'cron_feedback_classification' | 'tts_generation' | 'crm_landing_analysis' | 'crm_mail_generation';
     generationId: string | null;
     modelRequested: string;
     modelUsed: string | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autokap",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "AI-powered CLI tool for capturing clean screenshots of websites",
   "type": "module",
   "main": "./dist/index.js",
@@ -231,7 +231,8 @@
     "satori": "^0.26.0",
     "wawoff2": "^2.0.1",
     "ws": "^8.20.0",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "cheerio": "^1.1.2"
   },
   "devDependencies": {
     "@types/node": "^25.3.3",