autokap 1.0.6 → 1.0.8

package/dist/video-observation.d.ts

@@ -0,0 +1,36 @@
+import { Browser } from './browser.js';
+import type { BrowserVideoVerificationBundle } from './browser.js';
+import type { BrowserSessionStorageState, BrowserStorageState, CapturePageIdentity, InteractiveElement, VideoAgentConfig, VideoObservationSnapshot, VideoPageSignals } from './types.js';
+export declare function buildVideoObservationSummary(params: {
+    url: string;
+    title: string;
+    accessibilityTree: string;
+    interactiveElements: InteractiveElement[];
+    pageSignals: VideoPageSignals;
+    maxAccessibilityChars?: number;
+    maxElements?: number;
+    maxVisibleTextChars?: number;
+}): string;
+export declare function buildVideoObservationSnapshot(params: {
+    coherenceKey?: string;
+    interactiveElements: InteractiveElement[];
+    pageSignals: VideoPageSignals;
+    pageIdentity?: CapturePageIdentity | null;
+}): VideoObservationSnapshot;
+export declare function captureVideoObservationSummary(source: Browser | BrowserVideoVerificationBundle, options?: {
+    maxAccessibilityChars?: number;
+    maxElements?: number;
+    maxVisibleTextChars?: number;
+}): Promise<string>;
+export interface PlanningObservation {
+    summary: string;
+    screenshot?: Buffer;
+    snapshot?: VideoObservationSnapshot;
+}
+export declare function observePlanningContext(config: Pick<VideoAgentConfig, 'url' | 'viewport' | 'outputScale' | 'lang' | 'theme' | 'abortSignal'> & {
+    startUrl?: string;
+    storageState?: BrowserStorageState;
+    sessionStorage?: BrowserSessionStorageState;
+    captureScreenshot?: boolean;
+    pageIdentity?: CapturePageIdentity | null;
+}): Promise<PlanningObservation>;

package/dist/video-observation.js

@@ -0,0 +1,192 @@
+import { Browser } from './browser.js';
+import { throwIfAborted } from './abort.js';
+import { dismissOverlaysWithLogging } from './overlay-utils.js';
+function summarizeObservationElements(elements, limit = 18) {
+    const rank = (element) => {
+        let score = 0;
+        if (element.visible)
+            score += 20;
+        if (element.visibilityState === 'full')
+            score += 12;
+        if (element.href)
+            score += 10;
+        if (element.role === 'link' || element.tag === 'a')
+            score += 8;
+        if (element.role === 'button' || element.tag === 'button')
+            score += 6;
+        if (element.inputType)
+            score += 5;
+        if (element.text)
+            score += Math.min(12, element.text.length / 6);
+        if (element.ariaLabel)
+            score += 4;
+        return score;
+    };
+    return [...elements]
+        .sort((a, b) => rank(b) - rank(a))
+        .slice(0, limit)
+        .map((element) => {
+        const label = element.text || element.ariaLabel || '(no label)';
+        const href = element.href ? ` href=${element.href}` : '';
+        return `- [${element.role || element.tag}] "${label}"${href}`;
+    });
+}
+function formatSignalList(title, values, maxItems = 8) {
+    if (values.length === 0)
+        return '';
+    return `${title}: ${values.slice(0, maxItems).join(' | ')}`;
+}
+function summarizeVariantControls(signals, limit = 8) {
+    return signals.variantControls
+        .slice(0, limit)
+        .map((control) => {
+        const options = control.options?.length
+            ? ` options=${control.options
+                .slice(0, 5)
+                .map((option) => `${option.selected ? '*' : ''}${option.label || option.value || '(empty)'}`)
+                .join(', ')}`
+            : '';
+        const href = control.href ? ` href=${control.href}` : '';
+        return `- [${control.kind}/${control.mechanism}] "${control.label}" selector=${control.selector}${href}${options}`;
+    });
+}
+function summarizeStorageHints(signals, limit = 8) {
+    return signals.storageHints
+        .slice(0, limit)
+        .map((hint) => `- [${hint.kind}] ${hint.storage}.${hint.key}=${hint.valueSample || '(empty)'}`);
+}
+export function buildVideoObservationSummary(params) {
+    const signalLines = [
+        `Observed URL: ${params.url}`,
+        `Observed title: ${params.title || '(untitled)'}`,
+        params.pageSignals.htmlLang ? `Observed html lang: ${params.pageSignals.htmlLang}` : '',
+        params.pageSignals.canonicalUrl ? `Canonical URL: ${params.pageSignals.canonicalUrl}` : '',
+        params.pageSignals.detectedTheme ? `Observed theme: ${params.pageSignals.detectedTheme}` : '',
+        formatSignalList('Headings', params.pageSignals.headings),
+        formatSignalList('Navigation labels', params.pageSignals.navLabels, 10),
+        formatSignalList('Breadcrumbs', params.pageSignals.breadcrumbLabels, 8),
+        formatSignalList('Locale hints', params.pageSignals.localeHints, 10),
+        formatSignalList('hreflang links', params.pageSignals.hreflangs, 8),
+        params.pageSignals.visibleText
+            ? `Visible text sample: ${params.pageSignals.visibleText.slice(0, params.maxVisibleTextChars ?? 500)}`
+            : '',
+    ].filter(Boolean);
+    const elementLines = summarizeObservationElements(params.interactiveElements, params.maxElements ?? 18);
+    const controlLines = summarizeVariantControls(params.pageSignals, 8);
+    const storageLines = summarizeStorageHints(params.pageSignals, 8);
+    const accessibilitySnippet = params.accessibilityTree.slice(0, params.maxAccessibilityChars ?? 3500);
+    return [
+        ...signalLines,
+        controlLines.length > 0 ? 'Likely variant controls:' : '',
+        ...controlLines,
+        storageLines.length > 0 ? 'Likely variant storage keys:' : '',
+        ...storageLines,
+        elementLines.length > 0 ? 'Visible interactive elements:' : '',
+        ...elementLines,
+        accessibilitySnippet ? `Accessibility tree snippet:\n${accessibilitySnippet}` : '',
+    ]
+        .filter(Boolean)
+        .join('\n');
+}
+export function buildVideoObservationSnapshot(params) {
+    return {
+        coherenceKey: params.coherenceKey,
+        interactiveElements: params.interactiveElements,
+        pageSignals: params.pageSignals,
+        pageIdentity: params.pageIdentity ?? null,
+    };
+}
+export async function captureVideoObservationSummary(source, options = {}) {
+    if (source instanceof Browser) {
+        const [pageState, pageSignals, pageTitle] = await Promise.all([
+            source.getPageState(),
+            source.capturePageSignals(),
+            source.currentPage.title().catch(() => ''),
+        ]);
+        return buildVideoObservationSummary({
+            url: source.currentPage.url(),
+            title: pageTitle,
+            accessibilityTree: pageState.accessibilityTree,
+            interactiveElements: pageState.interactiveElements,
+            pageSignals,
+            ...options,
+        });
+    }
+    return buildVideoObservationSummary({
+        url: source.url,
+        title: source.title,
+        accessibilityTree: source.accessibilityTree,
+        interactiveElements: source.interactiveElements,
+        pageSignals: source.pageSignals,
+        ...options,
+    });
+}
+export async function observePlanningContext(config) {
+    throwIfAborted(config.abortSignal, 'Video planning observation cancelled.');
+    const browser = await Browser.fromPool({
+        headed: false,
+        viewport: config.viewport,
+        deviceScaleFactor: config.outputScale,
+        lang: config.lang,
+        colorScheme: config.theme,
+        storageState: config.storageState,
+    });
+    try {
+        await browser.prepareSessionStorage(config.sessionStorage, { replace: true });
+        await browser.navigateTo(config.startUrl ?? config.url);
+        await dismissOverlaysWithLogging(browser, { context: 'planning observation' });
+        const [summary, screenshot] = await Promise.all([
+            captureVideoObservationSummary(browser, {
+                maxAccessibilityChars: 3500,
+                maxElements: 18,
+                maxVisibleTextChars: 500,
+            }),
+            config.captureScreenshot
+                ? browser.takeScreenshotForAI().catch(() => undefined)
+                : Promise.resolve(undefined),
+        ]);
+        const [interactiveElements, pageSignals] = await Promise.all([
+            browser.getInteractiveElements().catch(() => []),
+            browser.capturePageSignals().catch(() => ({
+                url: browser.currentPage.url(),
+                title: '',
+                htmlLang: '',
+                canonicalUrl: '',
+                hreflangs: [],
+                headings: [],
+                navLabels: [],
+                breadcrumbLabels: [],
+                visibleText: '',
+                localeHints: [],
+                detectedTheme: null,
+                preferredColorScheme: 'light',
+                authHints: {
+                    hasPasswordField: false,
+                    hasEmailField: false,
+                    hasAuthForm: false,
+                    loginButtons: [],
+                    logoutButtons: [],
+                    accountMenuLabels: [],
+                    accountLikeText: [],
+                },
+                variantControls: [],
+                storageHints: [],
+            })),
+        ]);
+        return {
+            summary,
+            screenshot,
+            snapshot: buildVideoObservationSnapshot({
+                interactiveElements,
+                pageSignals,
+                pageIdentity: config.pageIdentity ?? null,
+            }),
+        };
+    }
+    finally {
+        await browser.close().catch(() => {
+            // Best effort cleanup.
+        });
+    }
+}
+//# sourceMappingURL=video-observation.js.map

package/dist/video-planner.d.ts

@@ -0,0 +1,12 @@
+import type { StepUsage, VideoPlan } from './types.js';
+import { type VideoPlannerPromptOptions } from './video-prompts.js';
+export { observePlanningContext } from './video-observation.js';
+export interface PlannerResult {
+    plan: VideoPlan;
+    usage: StepUsage;
+}
+/**
+ * Phase 1 of the video pipeline: convert a natural-language script into a
+ * structured `VideoPlan` using a single LLM call in JSON mode.
+ */
+export declare function planFromScript(script: string, url: string, model: string, apiKey: string, maxRetries?: number, options?: VideoPlannerPromptOptions, abortSignal?: AbortSignal): Promise<PlannerResult>;