autokap 1.0.7 → 1.0.8

package/dist/legacy/tools.d.ts

@@ -0,0 +1,4 @@
+import type { ChatCompletionFunctionTool } from 'openai/resources/chat/completions';
+export declare const agentTools: ChatCompletionFunctionTool[];
+/** Restricted tool set for the element capture mini-agent */
+export declare const elementCaptureTools: ChatCompletionFunctionTool[];

package/dist/legacy/tools.js

@@ -0,0 +1,216 @@
+// Note: evaluation_previous_action and memory fields are no longer injected into
+// tool schemas. They are documented in the system prompt instead (see prompts.ts),
+// which saves tokens on every iteration.
+const akTypeEnum = [
+    'button', 'input', 'link', 'text', 'image', 'heading',
+    'list', 'table', 'container', 'icon', 'toggle', 'select',
+    'checkbox', 'radio', 'slider', 'tab', 'modal', 'nav', 'form',
+    'video', 'audio', 'iframe',
+];
+const semanticPatternEnum = [
+    'cookie-banner', 'modal', 'navbar', 'footer', 'hero',
+    'sidebar', 'dropdown', 'toast', 'tooltip', 'form', 'card',
+    'pricing-table', 'cta-group',
+];
+const semanticTraitEnum = [
+    'floating', 'overlay', 'sticky', 'fixed',
+    'above-fold', 'below-fold', 'full-width', 'scrollable',
+];
+export const agentTools = [
+    {
+        type: 'function',
+        function: {
+            name: 'tap',
+            description: 'Click an element identified by its AKTree nodeId.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    nodeId: {
+                        type: 'string',
+                        description: 'Stable AKTree node id, for example "ak_3f8a2c1b".',
+                    },
+                },
+                required: ['nodeId'],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'type',
+            description: 'Type text into a text-like control identified by AKTree nodeId.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    nodeId: {
+                        type: 'string',
+                        description: 'Stable AKTree node id of the target input.',
+                    },
+                    text: {
+                        type: 'string',
+                        description: 'Text to type. Use credential placeholders when available.',
+                    },
+                },
+                required: ['nodeId', 'text'],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'scroll',
+            description: 'Scroll the page or a scrollable AKTree container. Use direction/offset for manual scrolling, or centerOn to center another AKTree node inside the page or container.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    direction: {
+                        type: 'string',
+                        enum: ['up', 'down', 'left', 'right'],
+                        description: 'Scroll direction for manual scrolling. Optional when centerOn is provided.',
+                    },
+                    offset: {
+                        type: 'number',
+                        description: 'Optional scroll offset in pixels for manual scrolling, or bias in pixels from perfect center when centerOn is provided.',
+                    },
+                    target: {
+                        type: 'string',
+                        description: 'Optional AKTree node id of a scrollable container. Omit to scroll the page.',
+                    },
+                    centerOn: {
+                        type: 'string',
+                        description: 'Optional AKTree node id to bring to the center of the page or of the target container.',
+                    },
+                },
+                required: [],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'navigate_to',
+            description: 'Navigate directly to a URL on the current site when that is the fastest deterministic way to reach the target.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    url: {
+                        type: 'string',
+                        description: 'Absolute or site-relative URL to navigate to.',
+                    },
+                },
+                required: ['url'],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'press_key',
+            description: 'Press a keyboard key when the workflow requires Escape, Enter, Tab, or arrow keys.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        description: 'Keyboard key name, for example "Escape", "Enter", "Tab", "ArrowDown".',
+                    },
+                },
+                required: ['key'],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'capture',
+            description: 'Capture the current target and end the current run. When nodeId is provided, crop to that node bounds.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    nodeId: {
+                        type: 'string',
+                        description: 'Optional AKTree node id to crop. Omit for a page-level capture.',
+                    },
+                },
+                required: [],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'analyze_screenshot',
+            description: 'Ask the vision model a targeted question about the current screenshot when AKTree structure is insufficient.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    question: {
+                        type: 'string',
+                        description: 'Specific visual question to answer, for example "Is a modal blocking the page?" or "Does this dialog already show the target template selected?"',
+                    },
+                },
+                required: ['question'],
+            },
+        },
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'focus',
+            description: 'Return a filtered AKTree view using typed filters only.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    type: {
+                        type: 'array',
+                        items: { type: 'string', enum: [...akTypeEnum] },
+                        description: 'Optional node types to keep.',
+                    },
+                    semantic: {
+                        type: 'array',
+                        items: { type: 'string', enum: [...semanticPatternEnum] },
+                        description: 'Optional semantic patterns to keep.',
+                    },
+                    trait: {
+                        type: 'array',
+                        items: { type: 'string', enum: [...semanticTraitEnum] },
+                        description: 'Optional semantic traits to keep.',
+                    },
+                    interactive: {
+                        type: 'boolean',
+                        description: 'Optional interactive filter.',
+                    },
+                    visible: {
+                        type: 'boolean',
+                        description: 'Optional visibility filter.',
+                    },
+                    within: {
+                        type: 'string',
+                        description: 'Optional AKTree node id used as subtree scope.',
+                    },
+                    labelContains: {
+                        type: 'string',
+                        description: 'Case-insensitive substring match against node labels.',
+                    },
+                    maxDepth: {
+                        type: 'number',
+                        description: 'Optional maximum subtree depth.',
+                    },
+                    includeAncestors: {
+                        type: 'boolean',
+                        description: 'Include parent path to the root when true.',
+                    },
+                },
+                required: [],
+            },
+        },
+    },
+];
+/** Restricted tool set for the element capture mini-agent */
+export const elementCaptureTools = [
+    agentTools[7],
+    agentTools[2],
+    agentTools[6],
+    agentTools[5],
+];
+//# sourceMappingURL=tools.js.map

package/dist/legacy/video-agent.d.ts

@@ -0,0 +1,143 @@
+import { Browser } from '../browser.js';
+import type { VideoAgentConfig, VideoAgentResult, VideoPhase, InteractiveElement, VideoStep, VideoPlan, StepUsage, VideoStepAttemptTrace, VideoStepArtifact, VideoSelectorMemoryUpdate, VideoPageSignals, VideoObservationSnapshot, BrowserSessionStorageState, BrowserStorageState } from '../types.js';
+export type VideoLogCallback = (entry: {
+    level: 'info' | 'success' | 'error' | 'ai';
+    message: string;
+    timestamp: number;
+}) => void;
+export type VideoPhaseCallback = (phase: VideoPhase) => void;
+export type VideoStepCallback = (stepIndex: number, total: number, description: string, phase: 'dry_run' | 'recording') => void;
+export type VideoStepResultCallback = (stepIndex: number, ok: boolean, reason?: string) => void;
+export type VideoVariantPhase = 'preflight' | 'planning' | 'dry_run' | 'recording' | 'done' | 'failed';
+export interface VideoPreflightResult {
+    ok: boolean;
+    reason?: string;
+    code?: string;
+    detectedLang?: string | null;
+    detectedTheme?: 'light' | 'dark' | null;
+    finalUrl?: string;
+    storageState?: BrowserStorageState;
+    sessionStorage?: BrowserSessionStorageState;
+    pageSignals?: VideoPageSignals;
+    observationSummary?: string;
+    observationSnapshot?: VideoObservationSnapshot;
+    usage?: StepUsage[];
+}
+export type VideoPreflightCallback = (result: VideoPreflightResult) => void;
+export type VideoVariantPhaseCallback = (phase: VideoVariantPhase) => void;
+export type VideoStepAttemptCallback = (attempt: VideoStepAttemptTrace) => void;
+export type VideoStepArtifactCallback = (artifact: VideoStepArtifact) => void;
+export interface VideoRunCallbacks {
+    onLog?: VideoLogCallback;
+    onPhaseChange?: VideoPhaseCallback;
+    onVariantPhase?: VideoVariantPhaseCallback;
+    onStep?: VideoStepCallback;
+    onStepResult?: VideoStepResultCallback;
+    onStepAttempt?: VideoStepAttemptCallback;
+    onStepArtifact?: VideoStepArtifactCallback;
+    onPreflightResult?: VideoPreflightCallback;
+    /** Emit a live Playwright screenshot (base64 JPEG) for real-time preview in the UI. */
+    onScreenshot?: (base64: string) => void;
+}
+type VideoDryRunFailureSubphase = 'prepared_variant' | 'deterministic_exec' | 'verification_snapshot' | 'verification_llm' | 'repair_lane' | 'llm_fixer' | 'checkpoint_restore' | 'budget';
+export interface VideoDryRunFailure {
+    stepIndex: number;
+    subphase: VideoDryRunFailureSubphase;
+    reason: string;
+    resolvedTargetSummary?: string;
+}
+interface VideoStepCheckpoint {
+    url: string;
+    scrollX: number;
+    scrollY: number;
+}
+interface ViewportRectSnapshot {
+    top: number;
+    bottom: number;
+    left: number;
+    right: number;
+    width: number;
+    height: number;
+}
+export declare function restoreVideoStepCheckpoint(browser: Browser, checkpoint: VideoStepCheckpoint): Promise<void>;
+export declare function isRectMeaningfullyVisibleInViewport(params: {
+    rect: ViewportRectSnapshot;
+    viewportWidth: number;
+    viewportHeight: number;
+    topInset?: number;
+    bottomInset?: number;
+}): boolean;
+export declare function sanitizeVariantPrefixSteps(steps: VideoStep[], requestedLang?: string, requestedTheme?: 'light' | 'dark'): VideoStep[];
+export declare function evaluateVideoStepSafety(step: VideoStep, config: VideoAgentConfig, page: import('playwright').Page, currentUrl: string, interactiveElements: InteractiveElement[]): Promise<{
+    allowed: boolean;
+    reason?: string;
+}>;
+export declare function extractNamedScrollTarget(description?: string): string | null;
+export declare function inspectVideoTarget(page: import('playwright').Page, selector?: string, coordinates?: {
+    x: number;
+    y: number;
+}): Promise<InteractiveElement | null>;
+interface VerifyStepResult {
+    ok: boolean;
+    reason?: string;
+    suggestion?: string;
+    giveUp?: boolean;
+    usage: Partial<StepUsage>;
+}
+export declare function verifyVideoStepDeterministically(browser: Browser, step: VideoStep): Promise<VerifyStepResult | null>;
+export declare function runVariantPreflight(config: VideoAgentConfig, prefixPlan: VideoPlan | null, callbacks?: VideoRunCallbacks): Promise<VideoPreflightResult>;
+export declare function composeHybridPlan(basePlan: VideoPlan, variantPrefixPlan: VideoPlan | null): VideoPlan;
+export declare function createBasePlan(config: VideoAgentConfig): Promise<{
+    plan: VideoPlan;
+    usage: StepUsage;
+}>;
+export declare function createVariantPrefixPlan(config: VideoAgentConfig): Promise<{
+    plan: VideoPlan;
+    usage: StepUsage;
+}>;
+export declare function verifyAndPatchPlan(plan: VideoPlan, config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    success: true;
+    plan: VideoPlan;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+} | {
+    success: false;
+    error: string;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+    failedStepIndex?: number;
+    failedSubphase?: VideoDryRunFailureSubphase;
+}>;
+/**
+ * Backward-compatible helper: full planning + verify in one call.
+ */
+export declare function planAndVerify(config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    success: true;
+    plan: VideoPlan;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+} | {
+    success: false;
+    error: string;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+}>;
+/**
+ * Run phase 3 (recording only) using a pre-verified plan.
+ * Call after `planAndVerify()` to record across multiple lang/theme combinations.
+ */
+export declare function recordPlan(plan: VideoPlan, config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    videoPath: string | null;
+    durationMs: number;
+    stepsExecuted: number;
+    setupDurationSec: number;
+    usage: StepUsage[];
+}>;
+/**
+ * Run the full 3-phase video capture pipeline:
+ * 1. Planning: script → structured plan (LLM)
+ * 2. Dry-run: verify all steps work
+ * 3. Recording: execute with Bezier mouse + cursor overlay
+ */
+export declare function runVideoAgent(config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<VideoAgentResult>;
+export {};