autokap 1.0.6 → 1.0.8

package/dist/element-capture.js

@@ -0,0 +1,522 @@
+import OpenAI from 'openai';
+import { elementCaptureTools } from './tools.js';
+import { buildElementSystemPrompt, buildElementIterationMessage } from './prompts.js';
+import { logger } from './logger.js';
+import { getPostHog, DISTINCT_ID } from './posthog.js';
+import { isAbortError, throwIfAborted } from './abort.js';
+import { callVisionCapableModel } from './vision-model.js';
+import { zdrParam } from './provider-config.js';
+const MAX_ELEMENT_ITERATIONS = 8;
+const ELEMENT_CAPTURE_TEMPERATURE = 0;
+export function isLooseElementCaptureRejectionReason(reason) {
+    if (!reason)
+        return false;
+    return /(too loose|too broad|tighten the frame|tighten the crop|significant unrelated surrounding content|large amount of unrelated page|minimal surrounding context|surrounding context|adjacent elements|wrong framing)/i
+        .test(reason);
+}
+export function outscaleAddsPadding(outscale) {
+    if (!outscale)
+        return false;
+    return [
+        outscale.padding,
+        outscale.paddingTop,
+        outscale.paddingRight,
+        outscale.paddingBottom,
+        outscale.paddingLeft,
+        outscale.paddingPercent,
+    ].some((value) => typeof value === 'number' && value > 0);
+}
+function buildTightOutscale(outscale) {
+    return {
+        ...outscale,
+        padding: 0,
+        paddingTop: 0,
+        paddingRight: 0,
+        paddingBottom: 0,
+        paddingLeft: 0,
+        paddingPercent: 0,
+    };
+}
+export function buildVerificationOutscale(outscale) {
+    return buildTightOutscale(outscale ?? {});
+}
+const elementVerificationTools = [
+    {
+        type: 'function',
+        function: {
+            name: 'accept_capture',
+            description: 'Approve the isolated element screenshot when the requested element/component is fully present and well framed.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    reason: {
+                        type: 'string',
+                        description: 'Short explanation of why the crop is good.',
+                    },
+                },
+                required: ['reason'],
+                additionalProperties: false,
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'accept_with_note',
+            description: 'Approve the capture with a minor note. Use when the correct element IS shown and fully visible, but the framing includes some extra surrounding context that does not materially affect capture quality.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    reason: {
+                        type: 'string',
+                        description: 'Short note about the minor framing issue.',
+                    },
+                },
+                required: ['reason'],
+                additionalProperties: false,
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'retry_capture',
+            description: 'Reject the isolated element screenshot when the crop is wrong, clipped, too loose, or shows the wrong target.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    reason: {
+                        type: 'string',
+                        description: 'What is wrong and what needs to be fixed before capturing again.',
+                    },
+                },
+                required: ['reason'],
+                additionalProperties: false,
+            },
+        },
+    },
+];
+async function verifyElementCapture(client, model, element, screenshot, assessment, stepNumber, signal, fallbackModel, uploadImage) {
+    throwIfAborted(signal, 'Element capture verification cancelled.');
+    const imgUrl = uploadImage
+        ? await uploadImage(screenshot, 'image/png').catch(() => `data:image/png;base64,${screenshot.toString('base64')}`)
+        : `data:image/png;base64,${screenshot.toString('base64')}`;
+    const messages = [
+        {
+            role: 'system',
+            content: 'You verify isolated UI element screenshots. The screenshot shows EXACTLY the pixel region captured by the element\'s bounding box — every pixel visible is INSIDE that element. Nothing outside the element is included.\n\nYou have THREE tools:\n- **accept_capture** — the correct component is shown, fully visible, well-framed\n- **accept_with_note** — the correct component IS shown and fully visible, but the framing includes some extra surrounding context. Use this instead of rejecting when the target element is clearly present and complete.\n- **retry_capture** — ONLY for serious issues: wrong element entirely, clipped edges, tiny fragment, or overlay obstruction\n\nReject (retry_capture) ONLY for:\n1. The element is clipped — a side is cut off by the image edges\n2. The wrong element was captured — the content clearly does not match the description at all\n3. Only a tiny fragment is captured — e.g., just a heading when a full card was requested\n4. An overlay or modal is obscuring the element\n\nWhen the correct element is visible but framing is slightly loose, use accept_with_note — do NOT reject.\n\nCRITICAL: Since the screenshot IS the element\'s exact bounding box, there is no such thing as "adjacent cards" or "neighboring elements" visible in this image. What looks like a neighboring card is an internal preview thumbnail. What looks like surrounding context is internal structure. Only reject if the image edges clip the component or the wrong component is shown entirely.',
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: { url: imgUrl },
+                },
+                {
+                    type: 'text',
+                    text: `## Requested isolated capture
+Name: "${element.name}"
+Description: "${element.description}"
+Candidate assessment: "${assessment}"
+
+This verification image is the raw element crop before any user-requested outscale or extra margin is applied programmatically after validation.
+
+The screenshot IS the captured element — every pixel is inside its bounding box.
+
+Use accept_capture if: the correct component is shown, fully visible, well-framed.
+Use accept_with_note if: the correct component IS shown and fully visible, but framing is slightly loose with some extra context. This is still an approval.
+
+Use retry_capture ONLY if:
+- a side of the component is clipped by the image boundary
+- the content clearly does not match the description at all (completely wrong element)
+- only a tiny fragment is shown when a full component (card, panel, form) was requested
+- an overlay or sticky header is covering it
+
+Do NOT reject because:
+- the element contains screenshot previews, thumbnails, or mockups inside it
+- the internal layout has multiple visual sections that look like separate cards
+- content appears "above", "beside", or "around" other content within the image — this is the element's own internal layout, not neighboring page elements
+- the framing is slightly loose but the correct element is clearly present and complete`,
+                },
+            ],
+        },
+    ];
+    let response;
+    let usedModel = model;
+    try {
+        const visionResult = await callVisionCapableModel({
+            primaryModel: model,
+            fallbackModel,
+            callModel: (m) => client.chat.completions.create({
+                model: m,
+                messages,
+                tools: elementVerificationTools,
+                tool_choice: 'required',
+                temperature: 0,
+                max_tokens: 256,
+                provider: { ...zdrParam() },
+            }, { signal }),
+            onFallbackActivated: (m, reason) => logger.info(`Element verification vision fallback: ${m} (reason: ${reason})`),
+        });
+        response = visionResult.result;
+        usedModel = visionResult.model;
+    }
+    catch (err) {
+        if (isAbortError(err))
+            throw err;
+        logger.error(`Element verification failed: ${err.message}`);
+        return { verified: false, reason: 'Verification service unavailable', usage: null };
+    }
+    const usage = {
+        stepNumber,
+        stepType: 'element_capture',
+        generationId: response.id ?? null,
+        modelRequested: model,
+        modelUsed: response.model ?? usedModel,
+        promptTokens: response.usage?.prompt_tokens ?? null,
+        completionTokens: response.usage?.completion_tokens ?? null,
+        totalTokens: response.usage?.total_tokens ?? null,
+        imagesInPrompt: 1,
+    };
+    const toolCall = response.choices?.[0]?.message?.tool_calls?.[0];
+    if (!toolCall || !('function' in toolCall)) {
+        return { verified: false, reason: 'Verification returned no decision', usage };
+    }
+    const name = toolCall.function.name;
+    let args;
+    try {
+        args = JSON.parse(toolCall.function.arguments);
+    }
+    catch {
+        return { verified: false, reason: 'Verification returned invalid JSON', usage };
+    }
+    if (name === 'accept_capture' || name === 'accept_with_note') {
+        return { verified: true, reason: args.reason || undefined, usage };
+    }
+    return { verified: false, reason: args.reason || 'Verifier rejected', usage };
+}
+export async function captureIsolatedElement(browser, element, apiKey, model, options = {}) {
+    const { abortSignal, distinctId, fallbackModel, uploadImage } = options;
+    const client = new OpenAI({
+        baseURL: 'https://openrouter.ai/api/v1',
+        apiKey,
+        defaultHeaders: {
+            'HTTP-Referer': 'https://github.com/screenshot-agent',
+            'X-Title': 'Screenshot Agent',
+        },
+    });
+    logger.info(`Identifying element: "${element.name}" — "${element.description}"`);
+    const actionHistory = [];
+    const usageLog = [];
+    let stepCounter = 0;
+    let lastVerifierRejectedAsTooLoose = false;
+    let looseRejectionCount = 0;
+    try {
+        for (let iteration = 1; iteration <= MAX_ELEMENT_ITERATIONS; iteration++) {
+            throwIfAborted(abortSignal, `Element capture cancelled for "${element.name}".`);
+            // 1. Capture page state with AKTree
+            const pageState = await browser.getPageState();
+            const screenshotBuf = pageState.cleanScreenshot;
+            const serializedAKTree = pageState.serializedAKTree;
+            const screenshotUrl = uploadImage
+                ? await uploadImage(screenshotBuf, 'image/png').catch(() => `data:image/png;base64,${screenshotBuf.toString('base64')}`)
+                : `data:image/png;base64,${screenshotBuf.toString('base64')}`;
+            // 2. Build messages
+            const messages = [
+                { role: 'system', content: buildElementSystemPrompt(element.description) },
+                {
+                    role: 'user',
+                    content: buildElementIterationMessage({
+                        elementName: element.name,
+                        elementDescription: element.description,
+                        serializedAKTree,
+                        currentUrl: browser.currentPage.url(),
+                        iteration,
+                        maxIterations: MAX_ELEMENT_ITERATIONS,
+                        actionHistory: actionHistory.length > 0 ? actionHistory : undefined,
+                        viewport: browser.currentPage.viewportSize() ?? undefined,
+                        scrollInfo: {
+                            scrollY: pageState.scrollInfo.scrollY,
+                            scrollHeight: pageState.scrollInfo.scrollHeight,
+                            viewportHeight: browser.currentPage.viewportSize()?.height ?? 900,
+                        },
+                        screenshotUrl,
+                    }),
+                },
+            ];
+            // 3. Call LLM
+            let response;
+            let usedModel = model;
+            let callMessages = messages;
+            const MAX_COERCION_RETRIES = 2;
+            try {
+                for (let coercionAttempt = 1; coercionAttempt <= MAX_COERCION_RETRIES + 1; coercionAttempt++) {
+                    throwIfAborted(abortSignal, `Element capture cancelled for "${element.name}".`);
+                    const visionResult = await callVisionCapableModel({
+                        primaryModel: model,
+                        fallbackModel,
+                        callModel: (m) => client.chat.completions.create({
+                            model: m,
+                            messages: callMessages,
+                            tools: elementCaptureTools,
+                            tool_choice: 'required',
+                            temperature: ELEMENT_CAPTURE_TEMPERATURE,
+                            max_tokens: 1024,
+                            provider: { ...zdrParam() },
+                        }, { signal: abortSignal }),
+                        onFallbackActivated: (m, reason) => logger.info(`Element capture vision fallback activated: ${m} (reason: ${reason})`),
+                    });
+                    response = visionResult.result;
+                    usedModel = visionResult.model;
+                    const msg = response.choices?.[0]?.message;
+                    const hasTool = !!msg?.tool_calls?.[0] && 'function' in msg.tool_calls[0];
+                    if (hasTool || coercionAttempt > MAX_COERCION_RETRIES)
+                        break;
+                    const assistantContent = msg?.content;
+                    if (!assistantContent)
+                        break;
+                    logger.info(`Model returned text without tool call; coercing (attempt ${coercionAttempt}/${MAX_COERCION_RETRIES})...`);
+                    callMessages = [
+                        ...callMessages,
+                        { role: 'assistant', content: assistantContent },
+                        {
+                            role: 'user',
+                            content: 'You MUST call one of the available tools. Do not respond with text — select the most appropriate tool and call it now.',
+                        },
+                    ];
+                }
+                usageLog.push({
+                    stepNumber: ++stepCounter,
+                    stepType: 'element_capture',
+                    generationId: response.id ?? null,
+                    modelRequested: model,
+                    modelUsed: response.model ?? usedModel,
+                    promptTokens: response.usage?.prompt_tokens ?? null,
+                    completionTokens: response.usage?.completion_tokens ?? null,
+                    totalTokens: response.usage?.total_tokens ?? null,
+                    imagesInPrompt: 1,
+                });
+            }
+            catch (err) {
+                if (isAbortError(err))
+                    throw err;
+                logger.error(`Element capture API call failed: ${err.message}`);
+                actionHistory.push(`Iteration ${iteration}: API error — ${err.message}`);
+                continue;
+            }
+            const message = response.choices?.[0]?.message;
+            const toolCall = message?.tool_calls?.[0];
+            if (message?.content && !toolCall) {
+                logger.ai(message.content.slice(0, 200));
+            }
+            if (!toolCall || !('function' in toolCall)) {
+                logger.error(`No tool call at element iteration ${iteration}`);
+                continue;
+            }
+            const name = toolCall.function.name;
+            let args;
+            try {
+                args = JSON.parse(toolCall.function.arguments);
+            }
+            catch {
+                logger.error(`Invalid JSON in element tool arguments: ${toolCall.function.arguments}`);
+                continue;
+            }
+            // 4. Handle tool calls
+            // ── capture(nodeId?) ──
+            if (name === 'capture') {
+                const nodeId = typeof args.nodeId === 'string' ? args.nodeId : undefined;
+                const baseOutscale = element.outscale ?? { padding: element.padding ?? 0 };
+                if (!nodeId) {
+                    // Full page capture — unusual for element capture but handle it
+                    logger.info(`Element "${element.name}": capture() without nodeId — full page`);
+                    const buffer = await browser.takeScreenshot();
+                    const viewport = browser.currentPage.viewportSize();
+                    const { verified, reason: verifyReason, usage } = await verifyElementCapture(client, model, element, buffer, 'Full page capture', ++stepCounter, abortSignal, fallbackModel, uploadImage);
+                    if (usage)
+                        usageLog.push(usage);
+                    if (verified) {
+                        logger.success(`Element "${element.name}" captured (full page)`);
+                        return { element, success: true, buffer, assessment: verifyReason || 'Full page', usage: usageLog };
+                    }
+                    actionHistory.push(`Iteration ${iteration}: capture() full page rejected — ${verifyReason}. Specify a nodeId to crop to the target element.`);
+                    continue;
+                }
+                // Resolve nodeId to bounds
+                const entry = await browser.resolveAKNode(nodeId);
+                if (!entry) {
+                    logger.error(`Element "${element.name}": nodeId "${nodeId}" not found in AKTree`);
+                    actionHistory.push(`Iteration ${iteration}: capture("${nodeId}") — node not found. Re-read the AKTree and use a valid nodeId.`);
+                    continue;
+                }
+                logger.info(`Element "${element.name}": capture("${nodeId}") — ${entry.label} ${entry.bounds.w}x${entry.bounds.h}`);
+                // Check area threshold
+                const viewport = browser.currentPage.viewportSize();
+                if (viewport) {
+                    const captureArea = entry.bounds.w * entry.bounds.h;
+                    const viewportArea = viewport.width * viewport.height;
+                    const isLargeComponent = /\b(hero|header|footer|navigation|navbar|banner|full.?width|feature.?section|testimonial|above.the.fold)\b/i.test(element.description);
+                    const areaThreshold = isLargeComponent ? 0.85 : 0.70;
+                    if (captureArea > viewportArea * areaThreshold) {
+                        logger.info(`Element "${element.name}": node "${nodeId}" covers ${Math.round(captureArea / viewportArea * 100)}% of viewport — too large`);
+                        actionHistory.push(`Iteration ${iteration}: capture("${nodeId}") rejected — element covers >${Math.round(areaThreshold * 100)}% of viewport (${entry.bounds.w}x${entry.bounds.h}), likely a wrapper. Find a more specific child node with focus(within: "${nodeId}").`);
+                        continue;
+                    }
+                }
+                // Capture the node
+                try {
+                    const buffer = await browser.captureNode(nodeId);
+                    // Verify
+                    const { verified, reason: verifyReason, usage } = await verifyElementCapture(client, model, element, buffer, `Captured ${entry.label} (${nodeId})`, ++stepCounter, abortSignal, fallbackModel, uploadImage);
+                    if (usage)
+                        usageLog.push(usage);
+                    if (!verified) {
+                        lastVerifierRejectedAsTooLoose = isLooseElementCaptureRejectionReason(verifyReason);
+                        if (lastVerifierRejectedAsTooLoose) {
+                            looseRejectionCount++;
+                            // Accept after repeated loose-only rejections (likely false positive from verifier)
+                            if (looseRejectionCount >= 3) {
+                                logger.info(`Element "${element.name}": accepting after ${looseRejectionCount} loose-only rejections`);
+                                return {
+                                    element, success: true, buffer,
+                                    assessment: 'Accepted after repeated loose-only verifier rejections.',
+                                    usage: usageLog,
+                                };
+                            }
+                        }
+                        actionHistory.push(`Iteration ${iteration}: capture("${nodeId}") rejected by verifier — ${verifyReason}. ${lastVerifierRejectedAsTooLoose ? 'Try a more specific child node with focus(within: "' + nodeId + '").' : 'Try a different nodeId.'}`);
+                        continue;
+                    }
+                    lastVerifierRejectedAsTooLoose = false;
+                    // Apply outscale if needed
+                    const finalBuffer = outscaleAddsPadding(baseOutscale)
+                        ? await browser.screenshotByRegion({ x: entry.bounds.x, y: entry.bounds.y, width: entry.bounds.w, height: entry.bounds.h }, baseOutscale)
+                        : buffer;
+                    logger.success(`Element "${element.name}" captured via nodeId "${nodeId}"`);
+                    getPostHog().capture({
+                        distinctId: distinctId ?? DISTINCT_ID,
+                        event: 'element_capture_succeeded',
+                        properties: { element_name: element.name, method: 'nodeId', nodeId, iterations: iteration },
+                    });
+                    return {
+                        element, success: true, buffer: finalBuffer,
+                        assessment: verifyReason || `Captured ${entry.label}`,
+                        usage: usageLog,
+                    };
+                }
+                catch (err) {
+                    if (isAbortError(err))
+                        throw err;
+                    logger.error(`Element "${element.name}": capture("${nodeId}") failed — ${err.message}`);
+                    actionHistory.push(`Iteration ${iteration}: capture("${nodeId}") error — ${err.message}`);
+                    continue;
+                }
+            }
+            // ── focus(query) ──
+            if (name === 'focus') {
+                const query = args;
+                try {
+                    const focusResult = await browser.focusTree(query);
+                    logger.info(`Element "${element.name}": focus → ${focusResult.matches.length} match(es)`);
+                    actionHistory.push(`Iteration ${iteration}: focus(${JSON.stringify(args)}) → ${focusResult.matches.length} match(es)\n${focusResult.serialized.slice(0, 800)}`);
+                }
+                catch (err) {
+                    logger.error(`Element "${element.name}": focus failed — ${err.message}`);
+                    actionHistory.push(`Iteration ${iteration}: focus error — ${err.message}`);
+                }
+                continue;
+            }
+            // ── scroll ──
+            if (name === 'scroll') {
+                const centerOn = typeof args.centerOn === 'string' ? args.centerOn : undefined;
+                const target = typeof args.target === 'string' ? args.target : undefined;
+                const direction = args.direction ?? 'down';
+                const offset = typeof args.offset === 'number' ? args.offset : undefined;
+                if (centerOn) {
+                    logger.info(`Element "${element.name}": scroll centerOn="${centerOn}"`);
+                    try {
+                        await browser.centerNodeInView(centerOn, { containerNodeId: target, offset });
+                        await browser.wait(300);
+                        actionHistory.push(`Iteration ${iteration}: scroll(centerOn="${centerOn}")`);
+                    }
+                    catch (err) {
+                        actionHistory.push(`Iteration ${iteration}: scroll centerOn error — ${err.message}`);
+                    }
+                }
+                else {
+                    const amount = offset ?? 500;
+                    logger.info(`Element "${element.name}": scroll ${direction} ${amount}px`);
+                    await browser.scroll(direction, amount);
+                    await browser.wait(500);
+                    actionHistory.push(`Iteration ${iteration}: scroll(${direction}, ${amount}px)`);
+                }
+                continue;
+            }
+            // ── analyze_screenshot ──
+            if (name === 'analyze_screenshot') {
+                const question = args.question || 'Describe what you see';
+                logger.info(`Element "${element.name}": analyze_screenshot — "${question}"`);
+                try {
+                    const analysisResponse = await client.chat.completions.create({
+                        model,
+                        messages: [
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'image_url', image_url: { url: screenshotUrl } },
+                                    { type: 'text', text: question },
+                                ],
+                            },
+                        ],
+                        max_tokens: 256,
+                        provider: { ...zdrParam() },
+                    }, { signal: abortSignal });
+                    const answer = analysisResponse.choices?.[0]?.message?.content || '(no answer)';
+                    actionHistory.push(`Iteration ${iteration}: analyze_screenshot("${question}") → ${answer.slice(0, 300)}`);
+                }
+                catch (err) {
+                    if (isAbortError(err))
+                        throw err;
+                    actionHistory.push(`Iteration ${iteration}: analyze_screenshot error — ${err.message}`);
+                }
+                continue;
+            }
+            // Unknown tool
+            logger.warn(`Element "${element.name}": unknown tool "${name}"`);
+            actionHistory.push(`Iteration ${iteration}: unknown tool "${name}" — use focus, scroll, analyze_screenshot, or capture.`);
+        }
+        // Max iterations reached — no coordinate fallback needed since AKTree provides bounds directly
+        logger.error(`Element "${element.name}": max iterations reached`);
+        getPostHog().capture({
+            distinctId: distinctId ?? DISTINCT_ID,
+            event: 'element_capture_failed',
+            properties: {
+                element_name: element.name,
+                reason: 'Max iterations reached',
+                iterations: MAX_ELEMENT_ITERATIONS,
+                failure_type: 'max_iterations',
+            },
+        });
+        return {
+            element,
+            success: false,
+            buffer: Buffer.alloc(0),
+            assessment: 'Max iterations reached for element identification',
+            usage: usageLog,
+        };
+    }
+    catch (err) {
+        if (isAbortError(err))
+            throw err;
+        logger.error(`Element capture unexpected error: ${err.message}`);
+        return {
+            element,
+            success: false,
+            buffer: Buffer.alloc(0),
+            assessment: `Unexpected error: ${err.message}`,
+            usage: usageLog,
+        };
+    }
+}
+//# sourceMappingURL=element-capture.js.map

package/dist/env-validation.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Startup validation for required environment variables.
+ * Call immediately after loading .env to fail fast with a clear message.
+ */
+export declare function validateRequiredEnv(): void;

package/dist/env-validation.js

@@ -0,0 +1,29 @@
+/**
+ * Startup validation for required environment variables.
+ * Call immediately after loading .env to fail fast with a clear message.
+ */
+const REQUIRED_ENV_VARS = [
+    'SUPABASE_SERVICE_ROLE_KEY',
+    'OPENROUTER_API_KEY',
+];
+const REQUIRED_WITH_PUBLIC_FALLBACK = [
+    { key: 'SUPABASE_URL', fallback: 'NEXT_PUBLIC_SUPABASE_URL' },
+];
+export function validateRequiredEnv() {
+    const missing = [];
+    for (const key of REQUIRED_ENV_VARS) {
+        if (!process.env[key]) {
+            missing.push(key);
+        }
+    }
+    for (const { key, fallback } of REQUIRED_WITH_PUBLIC_FALLBACK) {
+        if (!process.env[key] && !process.env[fallback]) {
+            missing.push(`${key} (or ${fallback})`);
+        }
+    }
+    if (missing.length > 0) {
+        console.error(`\n[AutoKap] Missing required environment variables:\n${missing.map((v) => `  - ${v}`).join('\n')}\n\nCopy .env.example to .env and fill in the values.\n`);
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=env-validation.js.map