autokap 1.0.2 → 1.0.4

package/dist/video-planner.js

@@ -1,500 +0,0 @@
-import OpenAI from 'openai';
-import { buildVideoPlannerSystemPrompt, buildVideoPlannerUserMessage, } from './video-prompts.js';
-export { observePlanningContext } from './video-observation.js';
-import { throwIfAborted } from './abort.js';
-import { extractStepUsage } from './llm-usage.js';
-function createClient(apiKey) {
-    return new OpenAI({
-        baseURL: 'https://openrouter.ai/api/v1',
-        apiKey,
-        defaultHeaders: {
-            'HTTP-Referer': 'https://github.com/screenshot-agent',
-            'X-Title': 'Screenshot Agent',
-        },
-    });
-}
-function isCoordinates(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const candidate = value;
-    return typeof candidate.x === 'number' && typeof candidate.y === 'number';
-}
-function containsInternalAutomationSelector(selector) {
-    return typeof selector === 'string' && /\[data-ak-[^\]]+\]|data-ak-interactive-index/i.test(selector);
-}
-function normalizeTarget(target) {
-    if (!target)
-        return undefined;
-    const selectorAlternates = Array.isArray(target.selectorAlternates)
-        ? target.selectorAlternates
-            .filter((selector) => typeof selector === 'string' && selector.trim().length > 0)
-            .filter((selector) => !containsInternalAutomationSelector(selector))
-        : [];
-    const selector = typeof target.selector === 'string' && target.selector.trim().length > 0 && !containsInternalAutomationSelector(target.selector)
-        ? target.selector.trim()
-        : undefined;
-    const normalized = {
-        ...target,
-        selector,
-        selectorAlternates,
-        coordinates: isCoordinates(target.coordinates) ? target.coordinates : undefined,
-        index: typeof target.index === 'number' ? target.index : undefined,
-    };
-    if (!normalized.selector && selectorAlternates.length > 0) {
-        normalized.selector = selectorAlternates[0];
-    }
-    return normalized;
-}
-function isPageExpectation(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const expectation = value;
-    if (Array.isArray(expectation.selectors) && expectation.selectors.some((selector) => containsInternalAutomationSelector(selector))) {
-        return false;
-    }
-    const listFields = ['urlPatterns', 'titlePatterns', 'textPatterns', 'navPatterns', 'breadcrumbPatterns', 'selectors']
-        .some((field) => Array.isArray(expectation[field]) && expectation[field].length > 0);
-    const scalarFields = typeof expectation.locale === 'string' || expectation.theme === 'light' || expectation.theme === 'dark';
-    return listFields || scalarFields;
-}
-function validateStep(step) {
-    if (!step || typeof step !== 'object')
-        return false;
-    const s = step;
-    if (typeof s.id !== 'string' || !s.id)
-        return false;
-    if (typeof s.type !== 'string')
-        return false;
-    if (typeof s.description !== 'string' || !s.description)
-        return false;
-    if (s.expectedPageAfter !== undefined && !isPageExpectation(s.expectedPageAfter))
-        return false;
-    const target = normalizeTarget(s.target);
-    const toTarget = normalizeTarget(s.toTarget);
-    const hasSelector = typeof s.selector === 'string' && !!s.selector.trim() && !containsInternalAutomationSelector(s.selector);
-    const hasCoordinates = isCoordinates(s.coordinates);
-    const hasTarget = !!(target?.selector || target?.coordinates);
-    switch (s.type) {
-        case 'navigate':
-            return typeof s.url === 'string' && !!s.url;
-        case 'dismiss_overlays':
-            return true;
-        case 'select_option':
-            return (hasSelector || hasTarget) && ((typeof s.optionLabel === 'string' && !!s.optionLabel.trim())
-                || (typeof s.optionValue === 'string' && !!s.optionValue.trim())
-                || typeof s.optionIndex === 'number');
-        case 'click':
-        case 'hover':
-        case 'highlight':
-            return hasSelector || hasCoordinates || hasTarget;
-        case 'type':
-            return (hasSelector || hasCoordinates || hasTarget) && typeof s.text === 'string';
-        case 'drag': {
-            const hasTargetSelector = typeof s.toSelector === 'string' && !!s.toSelector.trim() && !containsInternalAutomationSelector(s.toSelector);
-            const hasTargetCoordinates = isCoordinates(s.toCoordinates);
-            const hasStructuredDestination = !!(toTarget?.selector || toTarget?.coordinates);
-            return (hasSelector || hasCoordinates || hasTarget) && (hasTargetSelector || hasTargetCoordinates || hasStructuredDestination);
-        }
-        case 'scroll':
-            if (hasSelector || hasTarget)
-                return true;
-            return ((s.direction === 'up' || s.direction === 'down' || s.direction === 'left' || s.direction === 'right')
-                && typeof s.amount === 'number');
-        case 'wait':
-            return typeof s.waitMs === 'number' && s.waitMs >= 0;
-        case 'key':
-            return typeof s.key === 'string' && !!s.key;
-        case 'assert_url':
-            return typeof s.urlPattern === 'string' && !!s.urlPattern;
-        case 'assert_text':
-            return typeof s.text === 'string' && !!s.text;
-        case 'assert_element':
-            return hasSelector || hasTarget;
-        case 'assert_page': {
-            return isPageExpectation(s.pageExpectation);
-        }
-        default:
-            return false;
-    }
-}
-function normalizePageExpectation(expectation) {
-    if (!expectation)
-        return undefined;
-    return {
-        ...expectation,
-        minConfidence: expectation.minConfidence ?? 0.65,
-    };
-}
-function normalizeStep(step, index) {
-    const target = normalizeTarget(step.target);
-    const toTarget = normalizeTarget(step.toTarget);
-    const selector = step.selector ?? target?.selector ?? target?.selectorAlternates?.join(', ');
-    const coordinates = step.coordinates ?? target?.coordinates;
-    const toSelector = step.toSelector ?? toTarget?.selector ?? toTarget?.selectorAlternates?.join(', ');
-    const toCoordinates = step.toCoordinates ?? toTarget?.coordinates;
-    const baseStep = {
-        ...step,
-        id: step.id || `step-${index + 1}`,
-        recordingIntent: step.recordingIntent ?? 'visible',
-        target,
-        selector,
-        coordinates,
-        toTarget,
-        toSelector,
-        toCoordinates,
-        expectedPageAfter: normalizePageExpectation(step.expectedPageAfter),
-    };
-    if (step.type === 'assert_url') {
-        return {
-            ...baseStep,
-            matchMode: step.matchMode ?? 'contains',
-            timeoutMs: step.timeoutMs ?? 6000,
-        };
-    }
-    if (step.type === 'assert_text') {
-        return {
-            ...baseStep,
-            matchMode: step.matchMode ?? 'contains',
-            timeoutMs: step.timeoutMs ?? 6000,
-        };
-    }
-    if (step.type === 'assert_element') {
-        return {
-            ...baseStep,
-            state: step.state ?? 'visible',
-            timeoutMs: step.timeoutMs ?? 6000,
-        };
-    }
-    if (step.type === 'assert_page') {
-        return {
-            ...baseStep,
-            timeoutMs: step.timeoutMs ?? 6000,
-            pageExpectation: normalizePageExpectation(step.pageExpectation),
-        };
-    }
-    return baseStep;
-}
-function normalizeSemanticToken(value) {
-    return (value ?? '')
-        .normalize('NFD')
-        .replace(/[\u0300-\u036f]/g, '')
-        .replace(/\s+/g, ' ')
-        .trim()
-        .toLowerCase();
-}
-function slugifySemanticToken(value) {
-    return normalizeSemanticToken(value)
-        .replace(/[^a-z0-9]+/g, '-')
-        .replace(/^-+|-+$/g, '');
-}
-function extractDescriptionEntityPhrases(description) {
-    if (!description)
-        return [];
-    return Array.from(description.matchAll(/\b(?:open|click|highlight|hover|verify|scroll(?:\s+\w+){0,2}\s+to|go to|reveal)\s+(?:the\s+)?([a-z0-9][a-z0-9+.\-/]*(?:\s+[a-z0-9][a-z0-9+.\-/]*){0,4})\s+(?:page|section|link|button|tab|card)\b/gi))
-        .map((match) => match[1]?.trim() ?? '')
-        .filter((value) => value.length >= 3);
-}
-function isSpecificIntentPhrase(value) {
-    const normalized = normalizeSemanticToken(value);
-    if (!normalized)
-        return false;
-    const tokens = normalized.split(/\s+/).filter(Boolean);
-    return /\d/.test(normalized) || normalized.includes('-') || tokens.length >= 2;
-}
-function collectExpectationValues(expectation) {
-    if (!expectation)
-        return [];
-    return [
-        ...(expectation.urlPatterns ?? []),
-        ...(expectation.titlePatterns ?? []),
-        ...(expectation.textPatterns ?? []),
-        ...(expectation.navPatterns ?? []),
-        ...(expectation.breadcrumbPatterns ?? []),
-        ...(expectation.selectors ?? []),
-    ];
-}
-function collectStrictIntentTargets(step) {
-    const quoted = step.description
-        ? Array.from(step.description.matchAll(/['"]([^'"]{3,})['"]/g)).map((match) => match[1])
-        : [];
-    const phrases = [...new Set([
-            ...quoted,
-            ...extractDescriptionEntityPhrases(step.description),
-            ...collectExpectationValues(step.expectedPageAfter),
-            ...collectExpectationValues(step.pageExpectation),
-        ])].filter(isSpecificIntentPhrase);
-    const selectorSource = [
-        step.target?.label ?? '',
-        step.target?.href ?? '',
-        step.target?.selector ?? '',
-        ...(step.target?.selectorAlternates ?? []),
-        step.selector ?? '',
-        ...(collectExpectationValues(step.expectedPageAfter)),
-        ...(collectExpectationValues(step.pageExpectation)),
-    ].join(' ');
-    const slugs = [...new Set(selectorSource
-            .match(/[a-z0-9]+(?:-[a-z0-9]+)+/gi)?.map((value) => value.toLowerCase()) ?? [])].filter((value) => value.length >= 3 && (value.includes('-') || /\d/.test(value)));
-    return {
-        phrases,
-        slugs: [...new Set([...slugs, ...phrases.map(slugifySemanticToken)].filter(Boolean))],
-        strict: phrases.length > 0 || slugs.length > 0,
-    };
-}
-function selectorSupportsTargets(selector, step) {
-    if (!selector)
-        return false;
-    const targets = collectStrictIntentTargets(step);
-    if (!targets.strict)
-        return true;
-    const normalizedSelector = normalizeSemanticToken(selector);
-    return targets.phrases.some((phrase) => normalizedSelector.includes(normalizeSemanticToken(phrase)))
-        || targets.slugs.some((slug) => normalizedSelector.includes(slug));
-}
-function expectationSupportsTargets(expectation, step) {
-    if (!expectation)
-        return false;
-    const targets = collectStrictIntentTargets(step);
-    if (!targets.strict)
-        return true;
-    const haystack = normalizeSemanticToken(collectExpectationValues(expectation).join(' '));
-    return targets.phrases.some((phrase) => haystack.includes(normalizeSemanticToken(phrase)))
-        || targets.slugs.some((slug) => haystack.includes(slug));
-}
-function isNavigationContractCandidate(step) {
-    return step.type === 'navigate'
-        || step.type === 'click'
-        || step.type === 'select_option'
-        || step.type === 'type'
-        || step.type === 'key';
-}
-function deriveExpectedPageAfter(steps, index) {
-    for (let cursor = index + 1; cursor < Math.min(steps.length, index + 4); cursor += 1) {
-        const nextStep = steps[cursor];
-        if (nextStep.type === 'wait' || nextStep.type === 'dismiss_overlays') {
-            continue;
-        }
-        if (nextStep.type === 'assert_page') {
-            return normalizePageExpectation(nextStep.pageExpectation);
-        }
-        if (nextStep.type === 'assert_url' && nextStep.urlPattern) {
-            return normalizePageExpectation({
-                urlPatterns: [nextStep.urlPattern],
-                minConfidence: nextStep.matchMode === 'equals' ? 1 : 0.6,
-            });
-        }
-        break;
-    }
-    return undefined;
-}
-function enrichPlanRuntimeContracts(steps) {
-    return steps.map((step, index) => {
-        if (!isNavigationContractCandidate(step) || step.expectedPageAfter) {
-            return step;
-        }
-        const expectedPageAfter = deriveExpectedPageAfter(steps, index);
-        if (!expectedPageAfter)
-            return step;
-        return {
-            ...step,
-            expectedPageAfter,
-        };
-    });
-}
-function extractNamedScrollTarget(description) {
-    if (!description)
-        return null;
-    const quoted = description.match(/['"]([^'"]{3,})['"]/);
-    if (quoted)
-        return quoted[1];
-    const sectionMatch = description.match(/\bto (?:the\s+)?([a-z0-9][a-z0-9\s+.\-/'’]{2,50}) section\b/i);
-    return sectionMatch?.[1]?.trim() ?? null;
-}
-function lintPlanSemantics(plan, options = {}) {
-    const steps = plan.steps;
-    const visibleSteps = steps.filter((step) => step.recordingIntent !== 'prepare_only');
-    for (let i = 0; i < visibleSteps.length; i += 1) {
-        const step = visibleSteps[i];
-        if (i > 0 && step.type === 'navigate') {
-            return `Plan is invalid: visible step "${step.description}" uses direct navigate mid-flow. Recorded demos must navigate through the UI after the initial landing.`;
-        }
-    }
-    for (const step of steps) {
-        if (step.type === 'scroll') {
-            const namedTarget = extractNamedScrollTarget(step.description);
-            if (namedTarget && !step.selector) {
-                return `Plan is invalid: scroll step "${step.description}" targets a named section but has no selector. Use a selector-based scroll for named sections.`;
-            }
-            if (namedTarget && !selectorSupportsTargets(step.selector, step)) {
-                return `Plan is invalid: scroll step "${step.description}" targets a specific named section but its selector is too generic. Use a selector containing the target text or section slug.`;
-            }
-        }
-        if (step.type === 'click' || step.type === 'hover' || step.type === 'highlight' || step.type === 'assert_element') {
-            const targets = collectStrictIntentTargets(step);
-            if (!targets.strict)
-                continue;
-            const selectorOk = selectorSupportsTargets(step.selector, step);
-            const contractOk = step.type === 'click' && expectationSupportsTargets(step.expectedPageAfter, step);
-            if (!selectorOk && !contractOk && !step.coordinates) {
-                return `Plan is invalid: step "${step.description}" targets a specific page/element but its selector is too generic. Use exact product labels/routes or attach expectedPageAfter.`;
-            }
-        }
-        if ((step.type === 'click' || step.type === 'navigate' || step.type === 'select_option' || step.type === 'key')
-            && /\b(open|go to|navigate to|land on|arrive on)\b/i.test(step.description)
-            && /\b(page|pricing|dashboard|section|screen|tab)\b/i.test(step.description)
-            && !step.expectedPageAfter
-            && step.recordingIntent !== 'prepare_only') {
-            return `Plan is invalid: step "${step.description}" changes page/state but has no runtime destination contract. Add expectedPageAfter or a following assert_page/assert_url.`;
-        }
-    }
-    if (options.variant?.lang && options.mode === 'base') {
-        for (const step of steps) {
-            if (step.type === 'assert_text' && /\b(learn more|buy|hello|pricing|features)\b/i.test(step.text ?? '')) {
-                return `Plan is invalid: localized base plan still contains a likely English assert_text "${step.text}". Regenerate the plan in the prepared variant context.`;
-            }
-        }
-    }
-    return null;
-}
-function validatePlan(parsed) {
-    if (!parsed || typeof parsed !== 'object')
-        return false;
-    const obj = parsed;
-    if (typeof obj.title !== 'string' || !obj.title)
-        return false;
-    if (typeof obj.estimatedDurationSec !== 'number')
-        return false;
-    if (typeof obj.startUrl !== 'string' || !obj.startUrl)
-        return false;
-    if (!Array.isArray(obj.steps) || obj.steps.length === 0)
-        return false;
-    for (const step of obj.steps) {
-        if (!validateStep(step))
-            return false;
-    }
-    return true;
-}
-function normalizeAndValidatePlan(parsed, options = {}) {
-    const normalizedPlan = {
-        ...parsed,
-        steps: enrichPlanRuntimeContracts(parsed.steps.map((step, i) => normalizeStep(step, i))),
-    };
-    const semanticError = lintPlanSemantics(normalizedPlan, options);
-    if (semanticError) {
-        return { error: semanticError };
-    }
-    return { plan: normalizedPlan };
-}
-/**
- * Phase 1 of the video pipeline: convert a natural-language script into a
- * structured `VideoPlan` using a single LLM call in JSON mode.
- */
-export async function planFromScript(script, url, model, apiKey, maxRetries = 2, options = {}, abortSignal) {
-    const client = createClient(apiKey);
-    const systemPrompt = buildVideoPlannerSystemPrompt(options);
-    const userMessage = buildVideoPlannerUserMessage(script, url, options);
-    let lastError = '';
-    let lastResponse = null;
-    for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
-        throwIfAborted(abortSignal, 'Video planning cancelled.');
-        // Build user message — include screenshot if available for visual grounding
-        const userContent = [
-            { type: 'text', text: userMessage },
-        ];
-        if (options.screenshot) {
-            userContent.push({
-                type: 'image_url',
-                image_url: {
-                    url: `data:image/jpeg;base64,${options.screenshot.toString('base64')}`,
-                    detail: 'low',
-                },
-            });
-        }
-        const messages = [
-            { role: 'system', content: systemPrompt },
-            { role: 'user', content: options.screenshot ? userContent : userMessage },
-        ];
-        if (attempt > 1 && lastError) {
-            messages.push({
-                role: 'user',
-                content: `The previous response was invalid: ${lastError}. Please try again with a valid JSON plan.`,
-            });
-        }
-        const response = await client.chat.completions.create({
-            model,
-            messages,
-            max_tokens: 4000,
-            response_format: { type: 'json_object' },
-            provider: { zdr: true },
-        }, { signal: abortSignal });
-        lastResponse = response;
-        const content = response.choices?.[0]?.message?.content ?? '';
-        let parsed;
-        try {
-            parsed = JSON.parse(content);
-        }
-        catch (e) {
-            lastError = `JSON parse error: ${e.message}`;
-            continue;
-        }
-        if (validatePlan(parsed)) {
-            const normalized = normalizeAndValidatePlan(parsed, options);
-            if (!normalized.plan) {
-                lastError = normalized.error ?? 'Plan failed semantic validation.';
-                continue;
-            }
-            const usage = {
-                ...extractStepUsage(response, {
-                    stepNumber: 1,
-                    stepType: 'video_planning',
-                    modelRequested: model,
-                    imagesInPrompt: options.screenshot ? 1 : 0,
-                }),
-            };
-            return { plan: normalized.plan, usage };
-        }
-        lastError = 'Plan structure is invalid or missing required fields.';
-    }
-    const content = lastResponse?.choices?.[0]?.message?.content ?? '';
-    if (content) {
-        try {
-            const parsed = JSON.parse(content);
-            const steps = Array.isArray(parsed.steps) ? parsed.steps : [];
-            const normalizedSteps = steps
-                .map((s, i) => normalizeStep(s, i))
-                .filter((s) => validateStep(s));
-            const fallbackStep = {
-                id: 'step-1',
-                type: 'navigate',
-                description: 'Navigate to start URL',
-                url,
-            };
-            const plan = {
-                title: typeof parsed.title === 'string' ? parsed.title : 'Video demo',
-                estimatedDurationSec: typeof parsed.estimatedDurationSec === 'number' ? parsed.estimatedDurationSec : 30,
-                startUrl: typeof parsed.startUrl === 'string' ? parsed.startUrl : url,
-                steps: normalizedSteps.length > 0 ? normalizedSteps : [fallbackStep],
-            };
-            const normalized = normalizeAndValidatePlan(plan, options);
-            if (!normalized.plan) {
-                throw new Error(normalized.error ?? 'Plan failed semantic validation.');
-            }
-            if (!lastResponse) {
-                throw new Error('Video planner fallback lost the final model response.');
-            }
-            const usage = {
-                ...extractStepUsage(lastResponse, {
-                    stepNumber: 1,
-                    stepType: 'video_planning',
-                    modelRequested: model,
-                    imagesInPrompt: options.screenshot ? 1 : 0,
-                }),
-            };
-            return { plan: normalized.plan, usage };
-        }
-        catch {
-            // Fall through to hard error
-        }
-    }
-    throw new Error(`Failed to generate a valid video plan after ${maxRetries + 1} attempts. Last error: ${lastError}`);
-}
-//# sourceMappingURL=video-planner.js.map

package/dist/video-prompts.d.ts

@@ -1,37 +0,0 @@
-import type { ChatCompletionContentPart } from 'openai/resources/chat/completions';
-import type { LoginCredentials, VideoCursorTheme, VideoObservationSnapshot, VideoStep } from './types.js';
-export type VideoPlannerMode = 'full' | 'base' | 'variant_prefix' | 'clip';
-export interface VideoPlannerPromptVariant {
-    lang?: string;
-    theme?: 'light' | 'dark';
-    langInstructions?: string;
-    themeInstructions?: string;
-}
-export interface VideoPlannerPromptOptions {
-    mode?: VideoPlannerMode;
-    variant?: VideoPlannerPromptVariant;
-    credentials?: LoginCredentials;
-    observationSummary?: string;
-    observationSnapshot?: VideoObservationSnapshot;
-    /** Screenshot of the page for visual grounding. Sent as image to vision-capable models. */
-    screenshot?: Buffer;
-}
-export declare function buildVideoPromptContentParts(params: {
-    text: string;
-    imageUrl?: string;
-    cacheLayoutV2?: boolean;
-}): ChatCompletionContentPart[];
-export declare function buildStepFixerSystemPrompt(videoScript: string): string;
-export declare function buildStepFixerUserMessage(step: VideoStep, failureReason: string, suggestion: string, observationSummary?: string, observationSnapshot?: VideoObservationSnapshot): string;
-/**
- * JavaScript injected via `context.addInitScript()` to show a visible animated
- * cursor in Playwright video recordings (the native OS cursor is invisible).
- */
-export declare function buildCursorOverlayScript(theme?: VideoCursorTheme): string;
-export declare function buildVideoPlannerSystemPrompt(options?: VideoPlannerPromptOptions): string;
-export declare function buildVideoPlannerUserMessage(script: string, url: string, options?: VideoPlannerPromptOptions): string;
-export declare function buildVideoVerificationSystemPrompt(videoScript: string): string;
-export declare function buildVideoStepVerificationUserMessage(step: VideoStep, stepIndex: number, totalSteps: number, pageContext?: {
-    currentUrl?: string;
-    pageTitle?: string;
-}, observationSummary?: string, observationSnapshot?: VideoObservationSnapshot): string;