@nexus-cortex/executors 4.26.0

package/dist/implementations/addon/InteractWithSandboxTool.js

@@ -0,0 +1,367 @@
+import { BaseTool } from '../../base/BaseTool.js';
+import { CreateArtifactToolExecutor } from './CreateArtifactTool.js';
+import { visualBridge } from './VisualFeedbackBridge.js';
+import { broadcaster } from './SandboxEventBroadcaster.js';
+/**
+ * InteractWithSandboxTool - Interact with sandbox UI via Playwright
+ *
+ * This tool allows the model to interact with the sandbox UI programmatically:
+ * - Click buttons, links, elements
+ * - Type into inputs, textareas
+ * - Navigate to different URLs
+ * - Scroll, hover, select
+ * - Wait for animations/updates
+ * - Press keyboard shortcuts (Ctrl+V, Ctrl+S, etc.)
+ * - Zoom in/out
+ *
+ * Use Cases:
+ * 1. Testing - Click buttons to verify functionality
+ * 2. Form filling - Type into inputs to test forms
+ * 3. Navigation - Test multi-page flows
+ * 4. Data entry - Populate UI with test data
+ * 5. Visual regression - Capture UI states during interactions
+ * 6. Code pasting - Paste code snippets with Ctrl+V
+ * 7. Visual inspection - Zoom to see details
+ *
+ * Example:
+ * ```typescript
+ * // Model tests a form
+ * await interactWithSandbox({
+ *   sandboxId: "abc-123",
+ *   actions: [
+ *     { type: "type", selector: "#username", value: "testuser" },
+ *     { type: "type", selector: "#password", value: "pass123" },
+ *     { type: "click", selector: "#submit-btn" },
+ *     { type: "wait", duration: 1000 }
+ *   ],
+ *   returnFinalSnapshot: true
+ * });
+ *
+ * // Model pastes code into editor
+ * await interactWithSandbox({
+ *   sandboxId: "abc-123",
+ *   actions: [
+ *     { type: "click", selector: "#code-editor" },
+ *     { type: "keypress", key: "Ctrl+V" },  // Paste from clipboard
+ *     { type: "keypress", key: "Ctrl+S" },  // Save
+ *     { type: "zoom", zoomLevel: 1.5 }       // Zoom 150%
+ *   ]
+ * });
+ * ```
+ */
+export class InteractWithSandboxExecutor extends BaseTool {
+    constructor() {
+        const schema = {
+            type: 'object',
+            properties: {
+                sandboxId: {
+                    type: 'string',
+                    description: 'Unique ID of the sandbox to interact with'
+                },
+                actions: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            type: {
+                                type: 'string',
+                                enum: ['click', 'type', 'navigate', 'scroll', 'hover', 'select', 'wait', 'keypress', 'zoom'],
+                                description: 'Type of interaction'
+                            },
+                            selector: {
+                                type: 'string',
+                                description: 'CSS selector for the element (required for most actions)'
+                            },
+                            value: {
+                                type: 'string',
+                                description: 'Value to type or select (for type/select actions)'
+                            },
+                            coordinates: {
+                                type: 'object',
+                                properties: {
+                                    x: { type: 'number' },
+                                    y: { type: 'number' }
+                                },
+                                description: 'Screen coordinates (for click without selector)'
+                            },
+                            duration: {
+                                type: 'number',
+                                description: 'Duration in milliseconds (for wait action)'
+                            },
+                            key: {
+                                type: 'string',
+                                description: 'Key to press (for keypress action): Enter, Escape, Ctrl+V, etc.'
+                            },
+                            modifiers: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'Modifier keys (for keypress action): Control, Shift, Alt, Meta'
+                            },
+                            zoomLevel: {
+                                type: 'number',
+                                description: 'Zoom level (for zoom action): 1.0 = 100%, 1.5 = 150%, 0.5 = 50%'
+                            },
+                            deltaX: {
+                                type: 'number',
+                                description: 'Horizontal scroll delta (for scroll action)'
+                            },
+                            deltaY: {
+                                type: 'number',
+                                description: 'Vertical scroll delta (for scroll action)'
+                            }
+                        },
+                        required: ['type']
+                    },
+                    description: 'Sequence of actions to perform'
+                },
+                captureAfterEachAction: {
+                    type: 'boolean',
+                    description: 'Capture screenshot after each action (default: false, useful for debugging)'
+                },
+                returnFinalSnapshot: {
+                    type: 'boolean',
+                    description: 'Return visual snapshot after all actions complete (default: true)'
+                }
+            },
+            required: ['sandboxId', 'actions']
+        };
+        super('InteractWithSandbox', 'InteractWithSandbox', 'Interact with sandbox UI (click, type, navigate, etc.) via browser automation', schema);
+    }
+    validateToolParams(params) {
+        if (!params.sandboxId || params.sandboxId.trim().length === 0) {
+            return 'sandboxId is required';
+        }
+        if (!params.actions || params.actions.length === 0) {
+            return 'actions array cannot be empty';
+        }
+        // Validate each action
+        for (let i = 0; i < params.actions.length; i++) {
+            const action = params.actions[i];
+            if (!action) {
+                return `Action ${i}: action is undefined`;
+            }
+            if (!action.type) {
+                return `Action ${i}: type is required`;
+            }
+            // Validate action-specific requirements
+            if (action.type === 'click' && !action.selector && !action.coordinates) {
+                return `Action ${i}: click requires either selector or coordinates`;
+            }
+            if (action.type === 'type' && !action.selector) {
+                return `Action ${i}: type requires selector`;
+            }
+            if (action.type === 'type' && !action.value) {
+                return `Action ${i}: type requires value`;
+            }
+            if (action.type === 'navigate' && !action.value) {
+                return `Action ${i}: navigate requires value (URL)`;
+            }
+            if (action.type === 'scroll' && !action.coordinates && !action.selector) {
+                return `Action ${i}: scroll requires coordinates or selector`;
+            }
+            if (action.type === 'select' && (!action.selector || !action.value)) {
+                return `Action ${i}: select requires selector and value`;
+            }
+            if (action.type === 'wait' && !action.duration) {
+                return `Action ${i}: wait requires duration in milliseconds`;
+            }
+        }
+        return null;
+    }
+    async execute(params, signal) {
+        const startTime = Date.now();
+        const validationError = this.validateToolParams(params);
+        if (validationError) {
+            return this.createErrorResult(validationError);
+        }
+        try {
+            // Get sandbox session
+            const session = CreateArtifactToolExecutor.getActiveSandbox(params.sandboxId);
+            if (!session) {
+                return this.createErrorResult(`Sandbox not found: ${params.sandboxId}. It may have been stopped or never existed.`);
+            }
+            if (!session.url) {
+                return this.createErrorResult('Sandbox has no URL (not a web server)');
+            }
+            // Initialize visual bridge
+            await visualBridge.initialize();
+            // If a render trace is active, suppress the bracketing snapshots — captureSnapshot
+            // reloads the page (page.goto), which would add spurious mount-commits and could
+            // drown out the interaction's real re-render. Just drive the interaction on the live page.
+            const traceActive = await visualBridge.isTraceActive();
+            // Navigate to sandbox URL first (skip when tracing — page is already live)
+            const currentPage = visualBridge.page;
+            if (!traceActive && (!currentPage || currentPage.url() !== session.url)) {
+                await visualBridge.captureSnapshot(session.url);
+            }
+            // Execute actions
+            const actionResults = [];
+            const snapshots = [];
+            const captureAfterEach = params.captureAfterEachAction ?? false;
+            for (let i = 0; i < params.actions.length; i++) {
+                const action = params.actions[i];
+                if (!action)
+                    continue;
+                try {
+                    // Execute action
+                    await this.executeAction(action, session.url);
+                    actionResults.push(`[OK] Action ${i + 1}: ${action.type} ${action.selector || ''}`);
+                    // Emit interaction executed event
+                    broadcaster.emitInteraction(params.sandboxId, action.type, action.selector, true);
+                    // Capture snapshot if requested (suppressed during an active trace)
+                    if (captureAfterEach && !traceActive) {
+                        const snapshot = await visualBridge.captureSnapshot(session.url);
+                        snapshots.push({
+                            actionIndex: i,
+                            actionType: action.type,
+                            snapshot
+                        });
+                        // Emit screenshot captured event
+                        broadcaster.emitScreenshot(params.sandboxId, snapshot.screenshot, session.url);
+                    }
+                    // Check for abort signal
+                    if (signal.aborted) {
+                        throw new Error('Interaction aborted by user');
+                    }
+                }
+                catch (error) {
+                    actionResults.push(`[ERROR] Action ${i + 1}: ${action.type} - ${error.message}`);
+                    // Emit failed interaction event
+                    broadcaster.emitInteraction(params.sandboxId, action.type, action.selector, false);
+                    broadcaster.emitError(params.sandboxId, error);
+                    // Continue or stop on error?
+                    // For now, continue but report the error
+                    console.error(`Action ${i} failed:`, error);
+                }
+            }
+            // Capture final snapshot if requested
+            let finalSnapshot = null;
+            const returnFinalSnapshot = params.returnFinalSnapshot ?? true;
+            if (returnFinalSnapshot && !traceActive) {
+                finalSnapshot = await visualBridge.captureSnapshot(session.url);
+                session.visualSnapshot = finalSnapshot;
+                session.lastActivity = new Date();
+                // Emit screenshot captured event
+                broadcaster.emitScreenshot(params.sandboxId, finalSnapshot.screenshot, session.url);
+            }
+            // Format output
+            const output = this.formatInteractionOutput(session, params.actions, actionResults, snapshots, finalSnapshot);
+            return {
+                ...this.createSuccessResult(output),
+                metadata: {
+                    executionTime: Date.now() - startTime,
+                    sandboxId: params.sandboxId,
+                    actionsExecuted: params.actions.length,
+                    actionsSucceeded: actionResults.filter(r => r.startsWith('[OK]')).length,
+                    actionsFailed: actionResults.filter(r => r.startsWith('[ERROR]')).length,
+                    snapshotsCaptured: snapshots.length + (finalSnapshot ? 1 : 0)
+                }
+            };
+        }
+        catch (error) {
+            return this.createErrorResult(`Failed to interact with sandbox: ${error.message}`);
+        }
+    }
+    /**
+     * Execute a single interaction action
+     */
+    async executeAction(action, baseUrl) {
+        const interactionCommand = {
+            type: action.type,
+            selector: action.selector,
+            value: action.value,
+            coordinates: action.coordinates
+        };
+        switch (action.type) {
+            case 'click':
+                await visualBridge.interact(interactionCommand);
+                // Wait a bit for click effects
+                await this.wait(500);
+                break;
+            case 'type':
+                await visualBridge.interact(interactionCommand);
+                // Wait a bit for typing effects
+                await this.wait(300);
+                break;
+            case 'navigate':
+                // Navigate to URL (absolute or relative to baseUrl)
+                const targetUrl = action.value.startsWith('http')
+                    ? action.value
+                    : new URL(action.value, baseUrl).toString();
+                await visualBridge.interact({ type: 'navigate', value: targetUrl });
+                await this.wait(1000);
+                break;
+            case 'scroll':
+                await visualBridge.interact(interactionCommand);
+                await this.wait(300);
+                break;
+            case 'hover':
+                await visualBridge.interact({ type: 'click', selector: action.selector });
+                await this.wait(200);
+                break;
+            case 'select':
+                await visualBridge.interact(interactionCommand);
+                await this.wait(300);
+                break;
+            case 'wait':
+                await this.wait(action.duration);
+                break;
+            default:
+                throw new Error(`Unknown action type: ${action.type}`);
+        }
+    }
+    /**
+     * Wait helper
+     */
+    wait(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+    /**
+     * Format interaction output for model consumption
+     */
+    formatInteractionOutput(session, actions, actionResults, snapshots, finalSnapshot) {
+        const lines = [];
+        lines.push(`#  Sandbox Interaction: ${session.name}`);
+        lines.push('');
+        lines.push(`**Sandbox ID**: ${session.id}`);
+        lines.push(`**URL**: ${session.url}`);
+        lines.push(`**Actions Executed**: ${actions.length}`);
+        lines.push('');
+        lines.push('##  Action Results');
+        lines.push('');
+        actionResults.forEach(result => {
+            lines.push(result);
+        });
+        lines.push('');
+        // Show intermediate snapshots if captured
+        if (snapshots.length > 0) {
+            lines.push('---');
+            lines.push('');
+            lines.push('##  Intermediate Snapshots');
+            lines.push('');
+            snapshots.forEach((snap, idx) => {
+                lines.push(`### After Action ${snap.actionIndex + 1} (${snap.actionType})`);
+                lines.push('');
+                lines.push(`**Screenshot**: [Base64 PNG ${snap.snapshot.screenshot.length} bytes]`);
+                lines.push('');
+            });
+        }
+        // Show final snapshot
+        if (finalSnapshot) {
+            lines.push('---');
+            lines.push('');
+            lines.push('##  Final Visual State');
+            lines.push('');
+            lines.push(visualBridge.formatForModel(finalSnapshot));
+            lines.push('');
+        }
+        lines.push('---');
+        lines.push('');
+        lines.push('**Next Steps**:');
+        lines.push('- Inspect current state (Use inspect_sandbox)');
+        lines.push('- Continue interactions (Call interact_with_sandbox again)');
+        lines.push('- Edit code based on observations (Use write tool)');
+        return lines.join('\n');
+    }
+}
+//# sourceMappingURL=InteractWithSandboxTool.js.map

package/dist/implementations/addon/InteractWithSandboxTool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InteractWithSandboxTool.js","sourceRoot":"","sources":["../../../src/implementations/addon/InteractWithSandboxTool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAElD,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,YAAY,EAAsB,MAAM,2BAA2B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAuB3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,OAAO,2BAA4B,SAAQ,QAA+C;IAC9F;QACE,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAiB;YACvB,UAAU,EAAE;gBACV,SAAS,EAAE;oBACT,IAAI,EAAE,QAAiB;oBACvB,WAAW,EAAE,2CAA2C;iBACzD;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,OAAgB;oBACtB,KAAK,EAAE;wBACL,IAAI,EAAE,QAAiB;wBACvB,UAAU,EAAE;4BACV,IAAI,EAAE;gCACJ,IAAI,EAAE,QAAiB;gCACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC;gCAC5F,WAAW,EAAE,qBAAqB;6BACnC;4BACD,QAAQ,EAAE;gCACR,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,0DAA0D;6BACxE;4BACD,KAAK,EAAE;gCACL,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,mDAAmD;6BACjE;4BACD,WAAW,EAAE;gCACX,IAAI,EAAE,QAAiB;gCACvB,UAAU,EAAE;oCACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAiB,EAAE;oCAC9B,CAAC,EAAE,EAAE,IAAI,EAAE,QAAiB,EAAE;iCAC/B;gCACD,WAAW,EAAE,iDAAiD;6BAC/D;4BACD,QAAQ,EAAE;gCACR,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,4CAA4C;6BAC1D;4BACD,GAAG,EAAE;gCACH,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,iEAAiE;6BAC/E;4BACD,SAAS,EAAE;gCACT,IAAI,EAAE,OAAgB;gCACtB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAiB,EAAE;gCAClC,WAAW,EAAE,gEAAgE;6BAC9E;4BACD,SAAS,EAAE;gCACT,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,iEAAiE;6BAC/E;4BACD,MAAM,EAAE;gCACN,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,6CAA6C;6BAC3D;4BACD,MAAM,EAAE;gCACN,IAAI,EAAE,QAAiB;gCACvB,WAAW,EAAE,2CAA2C;6BACzD;yBACF;wBACD,QAAQ,EAAE,CAAC,MAAe,CAAC;qBAC5B;oBACD,WAAW,EAAE,gCAAgC;iBAC9C;gBACD,sBAAsB,EAAE;oBACtB,IAAI,EAAE,SAAkB;oBACxB,WAAW,EAAE,6EAA6E;iBAC3F;gBACD,mBAAmB,EAAE;oBACnB,IAAI,EAAE,SAAkB;oBACxB,WAAW,EAAE,mEAAmE;iBACjF;aACF;YACD,QAAQ,EAAE,CAAC,WAAoB,EAAE,SAAkB,CAAC;SACrD,CAAC;QAEF,KAAK,CACH,qBAAqB,EACrB,qBAAqB,EACrB,+EAA+E,EAC/E,MAAM,CACP,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,MAAiC;QAClD,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,OAAO,+BAA+B,CAAC;QACzC,CAAC;QAED,uBAAuB;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,UAAU,CAAC,uBAAuB,CAAC;YAC5C,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,UAAU,CAAC,oBAAoB,CAAC;YACzC,CAAC;YAED,wCAAwC;YACxC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvE,OAAO,UAAU,CAAC,iDAAiD,CAAC;YACtE,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC/C,OAAO,UAAU,CAAC,0BAA0B,CAAC;YAC/C,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC5C,OAAO,UAAU,CAAC,uBAAuB,CAAC;YAC5C,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChD,OAAO,UAAU,CAAC,iCAAiC,CAAC;YACtD,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACxE,OAAO,UAAU,CAAC,2CAA2C,CAAC;YAChE,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpE,OAAO,UAAU,CAAC,sCAAsC,CAAC;YAC3D,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC/C,OAAO,UAAU,CAAC,0CAA0C,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAiC,EAAE,MAAmB;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC;YACH,sBAAsB;YACtB,MAAM,OAAO,GAAG,0BAA0B,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE9E,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,IAAI,CAAC,iBAAiB,CAC3B,sBAAsB,MAAM,CAAC,SAAS,8CAA8C,CACrF,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC,uCAAuC,CAAC,CAAC;YACzE,CAAC;YAED,2BAA2B;YAC3B,MAAM,YAAY,CAAC,UAAU,EAAE,CAAC;YAEhC,mFAAmF;YACnF,iFAAiF;YACjF,2FAA2F;YAC3F,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,aAAa,EAAE,CAAC;YAEvD,2EAA2E;YAC3E,MAAM,WAAW,GAAI,YAAoB,CAAC,IAAI,CAAC;YAC/C,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxE,MAAM,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAClD,CAAC;YAED,kBAAkB;YAClB,MAAM,aAAa,GAAa,EAAE,CAAC;YACnC,MAAM,SAAS,GAAU,EAAE,CAAC;YAC5B,MAAM,gBAAgB,GAAG,MAAM,CAAC,sBAAsB,IAAI,KAAK,CAAC;YAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAEjC,IAAI,CAAC,MAAM;oBAAE,SAAS;gBAEtB,IAAI,CAAC;oBACH,iBAAiB;oBACjB,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBAE9C,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;oBAEpF,kCAAkC;oBAClC,WAAW,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;oBAElF,oEAAoE;oBACpE,IAAI,gBAAgB,IAAI,CAAC,WAAW,EAAE,CAAC;wBACrC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBACjE,SAAS,CAAC,IAAI,CAAC;4BACb,WAAW,EAAE,CAAC;4BACd,UAAU,EAAE,MAAM,CAAC,IAAI;4BACvB,QAAQ;yBACT,CAAC,CAAC;wBAEH,iCAAiC;wBACjC,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACjF,CAAC;oBAED,yBAAyB;oBACzB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,MAAO,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;oBAE5F,gCAAgC;oBAChC,WAAW,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;oBACnF,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,KAAc,CAAC,CAAC;oBAExD,6BAA6B;oBAC7B,yCAAyC;oBACzC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,sCAAsC;YACtC,IAAI,aAAa,GAAQ,IAAI,CAAC;YAC9B,MAAM,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;YAE/D,IAAI,mBAAmB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxC,aAAa,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAChE,OAAO,CAAC,cAAc,GAAG,aAAa,CAAC;gBACvC,OAAO,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;gBAElC,iCAAiC;gBACjC,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YACtF,CAAC;YAED,gBAAgB;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,uBAAuB,CACzC,OAAO,EACP,MAAM,CAAC,OAAO,EACd,aAAa,EACb,SAAS,EACT,aAAa,CACd,CAAC;YAEF,OAAO;gBACL,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBACnC,QAAQ,EAAE;oBACR,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBACrC,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;oBACtC,gBAAgB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;oBACxE,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;oBACxE,iBAAiB,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC9D;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,iBAAiB,CAAC,oCAAqC,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,MAAW,EAAE,OAAe;QACtD,MAAM,kBAAkB,GAAuB;YAC7C,IAAI,EAAE,MAAM,CAAC,IAAW;YACxB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC;QAEF,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,OAAO;gBACV,MAAM,YAAY,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;gBAChD,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM;YAER,KAAK,MAAM;gBACT,MAAM,YAAY,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;gBAChD,gCAAgC;gBAChC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM;YAER,KAAK,UAAU;gBACb,oDAAoD;gBACpD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC/C,CAAC,CAAC,MAAM,CAAC,KAAK;oBACd,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC9C,MAAM,YAAY,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,MAAM;YAER,KAAK,QAAQ;gBACX,MAAM,YAAY,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM;YAER,KAAK,OAAO;gBACV,MAAM,YAAY,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1E,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM;YAER,KAAK,QAAQ;gBACX,MAAM,YAAY,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM;YAER,KAAK,MAAM;gBACT,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACjC,MAAM;YAER;gBACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,IAAI,CAAC,EAAU;QACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,OAAY,EACZ,OAAc,EACd,aAAuB,EACvB,SAAgB,EAChB,aAAkB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,CAAC,IAAI,CAAC,2BAA2B,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,yBAAyB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC7B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,0CAA0C;QAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC9B,KAAK,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;gBAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC;gBACpF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,IAAI,aAAa,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QACzE,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAEjE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF"}

package/dist/implementations/addon/KeyframeDetector.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Keyframe Detector
+ *
+ * Intelligently detects when screenshots should be sent to vision API
+ * Uses DOM analysis, perceptual hashing, and event monitoring
+ * to minimize unnecessary API calls while maintaining accuracy.
+ *
+ * Features:
+ * - Navigation change detection
+ * - DOM mutation monitoring
+ * - Modal/alert detection
+ * - Visual error detection
+ * - Perceptual hash-based change detection
+ * - Event-driven keyframe triggers
+ */
+import { EventEmitter } from 'events';
+import { Page } from 'playwright';
+export interface KeyframeConfig {
+    domMutationThreshold?: number;
+    visualHashThreshold?: number;
+    modalCheckInterval?: number;
+    detectNavigation?: boolean;
+    detectDOMMutations?: boolean;
+    detectModals?: boolean;
+    detectErrors?: boolean;
+}
+export interface KeyframeTrigger {
+    type: 'navigation' | 'dom_mutation' | 'modal' | 'error' | 'user_request' | 'visual_change';
+    reason: string;
+    confidence: number;
+    metadata?: Record<string, any>;
+    timestamp: number;
+}
+export interface PageSnapshot {
+    url: string;
+    title: string;
+    domHash: string;
+    visualHash: string;
+    elementCount: number;
+    hasModal: boolean;
+    hasError: boolean;
+}
+/**
+ * KeyframeDetector - Smart detection of when to send screenshots to vision API
+ *
+ * Monitors page state and triggers keyframe captures only when significant
+ * changes occur, dramatically reducing API costs while maintaining accuracy.
+ *
+ * @example
+ * const detector = new KeyframeDetector(page, {
+ *   domMutationThreshold: 50,
+ *   visualHashThreshold: 0.15
+ * });
+ *
+ * detector.on('keyframe', (trigger: KeyframeTrigger) => {
+ *   console.log(`Keyframe needed: ${trigger.reason}`);
+ *   // Send screenshot to vision API
+ * });
+ *
+ * await detector.start();
+ */
+export declare class KeyframeDetector extends EventEmitter {
+    private page;
+    private config;
+    private isRunning;
+    private lastSnapshot;
+    private mutationCount;
+    private modalCheckInterval;
+    constructor(page: Page, config?: KeyframeConfig);
+    /**
+     * Start monitoring for keyframe triggers
+     */
+    start(): Promise<void>;
+    /**
+     * Stop monitoring
+     */
+    stop(): void;
+    /**
+     * Manually trigger keyframe (for user requests)
+     */
+    triggerKeyframe(reason: string, metadata?: Record<string, any>): void;
+    /**
+     * Capture current page snapshot
+     */
+    private captureSnapshot;
+    /**
+     * Set up navigation change detector
+     */
+    private setupNavigationDetector;
+    /**
+     * Set up DOM mutation detector
+     */
+    private setupDOMMutationDetector;
+    /**
+     * Set up modal detector (periodic check)
+     */
+    private setupModalDetector;
+    /**
+     * Set up error detector
+     */
+    private setupErrorDetector;
+    /**
+     * Detect if modal/dialog is present
+     */
+    private detectModal;
+    /**
+     * Detect if error is visible on page
+     */
+    private detectError;
+    /**
+     * Compute simple hash of content
+     */
+    private computeHash;
+    /**
+     * Compute perceptual hash difference between two snapshots
+     * Returns 0-1, where 0 is identical and 1 is completely different
+     */
+    computeVisualDifference(): Promise<number>;
+    /**
+     * Check if visual change threshold is exceeded
+     */
+    shouldSendToAPI(): Promise<{
+        should: boolean;
+        reason: string;
+        confidence: number;
+    }>;
+    /**
+     * Get current detector state
+     */
+    getState(): {
+        isRunning: boolean;
+        lastSnapshot: PageSnapshot | null;
+        config: Required<KeyframeConfig>;
+    };
+    /**
+     * Cleanup resources
+     */
+    destroy(): void;
+}
+//# sourceMappingURL=KeyframeDetector.d.ts.map

package/dist/implementations/addon/KeyframeDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"KeyframeDetector.d.ts","sourceRoot":"","sources":["../../../src/implementations/addon/KeyframeDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGlC,MAAM,WAAW,cAAc;IAE7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAG5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,YAAY,GAAG,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,cAAc,GAAG,eAAe,CAAC;IAC3F,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,SAAS,CAAkB;IAEnC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,aAAa,CAAa;IAClC,OAAO,CAAC,kBAAkB,CAA+C;gBAE7D,IAAI,EAAE,IAAI,EAAE,MAAM,GAAE,cAAmB;IAenD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B5B;;OAEG;IACH,IAAI,IAAI,IAAI;IAeZ;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAYrE;;OAEG;YACW,eAAe;IAmC7B;;OAEG;YACW,uBAAuB;IAyBrC;;OAEG;YACW,wBAAwB;IA+CtC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;YACW,kBAAkB;IA8BhC;;OAEG;YACW,WAAW;IAsBzB;;OAEG;YACW,WAAW;IAwBzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;;OAGG;IACG,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC;IAoBhD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAsCzF;;OAEG;IACH,QAAQ,IAAI;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;QAClC,MAAM,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;KAClC;IAQD;;OAEG;IACH,OAAO,IAAI,IAAI;CAKhB"}