@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/web/transformer.js

@@ -0,0 +1,41 @@
+import getShadowDOMInputAdapterScript from './scripts/shadowDOMInputAdapter';
+import logger from '@/logger';
+export class DOMTransformer {
+    initializedPages = new WeakSet(); // Track pages for which 'load' listener is set
+    constructor() { }
+    setActivePage(newPage) {
+        // Only add the 'load' listener if we haven't done so for this specific Page object instance.
+        if (!this.initializedPages.has(newPage)) {
+            newPage.on('load', async () => {
+                // Pass 'newPage' (the page that triggered the 'load' event) to setupScriptForPage.
+                await this.setupScriptForPage(newPage);
+            });
+            this.initializedPages.add(newPage); // Mark this Page object as having its 'load' listener set up.
+        }
+    }
+    async setupScriptForPage(targetPage) {
+        try {
+            // Check if a marker for the script already exists on the page for this load cycle.
+            const scriptAlreadyInjected = await targetPage.evaluate(() => {
+                return window.__magnitudeShadowDOMAdapterInjected === true;
+            }).catch(() => false); // If evaluate fails (e.g., page closed), assume not injected.
+            if (scriptAlreadyInjected) {
+                logger.trace('Select manager script already present on this page load.');
+                return;
+            }
+            // Get the script as a string from the separate file
+            const scriptFnString = getShadowDOMInputAdapterScript();
+            // Evaluate the script function in the browser and set the marker.
+            // We need to wrap it in a self-executing function
+            await targetPage.evaluate(`
+                (${scriptFnString})();
+                window.__magnitudeShadowDOMAdapterInjected = true;
+            `);
+            logger.trace(`Script injected into page: ${targetPage.url()}`);
+        }
+        catch (error) {
+            const url = targetPage.isClosed() ? '[closed page]' : targetPage.url();
+            logger.warn(`Error injecting script into ${url}: ${error.message}`);
+        }
+    }
+}

package/dist/web/types.d.ts

@@ -0,0 +1,32 @@
+export type Base64Image = `data:image/${'jpeg' | 'png' | 'gif'};base64,${string}`;
+export type WebAction = NavigateWebAction | ClickWebAction | TypeWebAction | ScrollWebAction | SwitchTabWebAction;
+export interface NavigateWebAction {
+    variant: 'load';
+    url: string;
+}
+export interface ClickWebAction {
+    variant: 'click';
+    x: number;
+    y: number;
+}
+export interface TypeWebAction {
+    variant: 'type';
+    x: number;
+    y: number;
+    content: string;
+}
+export interface ScrollWebAction {
+    variant: 'scroll';
+    x: number;
+    y: number;
+    deltaX: number;
+    deltaY: number;
+}
+export type SwitchTabWebAction = {
+    variant: 'tab';
+    index: number;
+};
+export interface PixelCoordinate {
+    x: number;
+    y: number;
+}

package/dist/web/types.js

@@ -0,0 +1,2 @@
+export {};
+// for back-compat, not all implemented yet

package/dist/web/util.d.ts

@@ -0,0 +1,3 @@
+export type KeypressIndicator = '<enter>' | '<tab>';
+export declare function parseTypeContent(content: string): (string | KeypressIndicator)[];
+export declare function renderMinimalAccessibilityTree(node: any): string;

package/dist/web/util.js

@@ -0,0 +1,81 @@
+export function parseTypeContent(content) {
+    // Parse content sequence potentially with special indicators <enter> or <tab> into components
+    const regex = /<enter>|<tab>/g;
+    const result = [];
+    let lastIndex = 0;
+    content.replace(regex, (match, offset) => {
+        // Grab the text before the match and trim surrounding whitespace
+        const text = content.slice(lastIndex, offset).trim();
+        if (text)
+            result.push(text);
+        // Add the matched special indicator
+        result.push(match);
+        lastIndex = offset + match.length;
+        return match;
+    });
+    // Add any remaining text after the last match
+    const remaining = content.slice(lastIndex).trim();
+    if (remaining)
+        result.push(remaining);
+    return result;
+}
+export function renderMinimalAccessibilityTree(node) {
+    // Escapes newlines for safe inline printing, and trims whitespace.
+    const sanitize = (text) => {
+        return String(text).trim().replace(/\n/g, '\\n');
+    };
+    // Inner recursive function to handle state like indentation
+    function recursiveFlatten(node, indent) {
+        const isTextLike = (n) => n && (n.role === 'StaticText' || n.role === 'text') && n.name && n.name.trim();
+        let childrenOutput = '';
+        if (node.children) {
+            const newChildren = [];
+            let currentTextBuffer = [];
+            for (const child of node.children) {
+                if (isTextLike(child)) {
+                    currentTextBuffer.push(sanitize(child.name));
+                }
+                else {
+                    if (currentTextBuffer.length > 0) {
+                        newChildren.push({ role: 'coalesced-text', name: currentTextBuffer.join(' ') });
+                        currentTextBuffer = [];
+                    }
+                    newChildren.push(child);
+                }
+            }
+            if (currentTextBuffer.length > 0) {
+                newChildren.push({ role: 'coalesced-text', name: currentTextBuffer.join(' ') });
+            }
+            for (const child of newChildren) {
+                if (child.role === 'coalesced-text') {
+                    childrenOutput += `${indent}  ${child.name}\n`;
+                }
+                else {
+                    childrenOutput += recursiveFlatten(child, indent + '  ');
+                }
+            }
+        }
+        const isBoringContainer = (node.role === 'generic' || node.role === 'div') && !node.name;
+        if (isBoringContainer) {
+            return childrenOutput;
+        }
+        let line = '';
+        if (node.role && !['StaticText', 'text', 'RootWebArea'].includes(node.role)) {
+            line = `${indent}[${node.role}]`;
+            if (node.name && sanitize(node.name)) {
+                line += ` ${sanitize(node.name)}`;
+            }
+            if (node.value !== undefined && sanitize(node.value)) {
+                line += ` (value: ${sanitize(node.value)})`;
+            }
+            if (node.checked !== undefined)
+                line += ` (checked: ${node.checked})`;
+            if (node.disabled)
+                line += ` (disabled)`;
+            line += '\n';
+        }
+        return line + childrenOutput;
+    }
+    const rawFlattened = recursiveFlatten(node, '');
+    return rawFlattened.split('\n').filter(line => line.trim() !== '').join('\n');
+}

package/dist/web/visualizer/cursor.d.ts

@@ -0,0 +1,16 @@
+import { Page } from "playwright";
+export declare class CursorVisual {
+    /**
+     * Manages the visual indicator for actions on a page
+     */
+    private page;
+    private visualElementId;
+    private lastPosition;
+    constructor();
+    setActivePage(page: Page): Promise<void>;
+    move(x: number, y: number): Promise<void>;
+    setupOnPage(): Promise<void>;
+    private _drawVisual;
+    hide(): Promise<void>;
+    show(): Promise<void>;
+}

package/dist/web/visualizer/cursor.js

@@ -0,0 +1,152 @@
+import { retryOnErrorIsSuccess } from "@/common";
+import logger from "@/logger";
+export class CursorVisual {
+    /**
+     * Manages the visual indicator for actions on a page
+     */
+    page;
+    visualElementId = 'action-visual-indicator';
+    lastPosition = null;
+    constructor() {
+        //this.page = page;
+    }
+    async setActivePage(page) {
+        this.page = page;
+        page.on('load', async () => {
+            await retryOnErrorIsSuccess(this.setupOnPage.bind(this), { mode: 'retry_all', delayMs: 200, retryLimit: 10 });
+        });
+        await retryOnErrorIsSuccess(this.setupOnPage.bind(this), { mode: 'retry_all', delayMs: 200, retryLimit: 5 });
+    }
+    async move(x, y) {
+        // Store the position
+        this.lastPosition = { x, y };
+        // Create or update the mouse pointer visual, showing the click effect
+        await this._drawVisual(x, y, false);
+        // The pointer visual takes 0.3s on the transition, but awaiting script evaluation does not wait for this to complete.
+        // So we wait 300ms manually.
+        await this.page.waitForTimeout(300);
+    }
+    async setupOnPage() {
+        if (this.lastPosition) {
+            // Redraw the visual without the click effect
+            await this._drawVisual(this.lastPosition.x, this.lastPosition.y, false);
+        }
+    }
+    // Internal method to handle the actual drawing logic
+    async _drawVisual(x, y, showClickEffect) {
+        try {
+            await this.page.evaluate(({ x, y, id, showClickEffect }) => {
+                // Use viewport coordinates directly (no scroll adjustment for fixed positioning)
+                const viewportX = x;
+                const viewportY = y;
+                // Document coordinates for the click effect circle (which uses absolute positioning)
+                const docX = x + window.scrollX;
+                const docY = y + window.scrollY;
+                // --- Create Expanding/Fading Circle (Optional) ---
+                if (showClickEffect) {
+                    const circle = document.createElement('div');
+                    circle.style.position = 'absolute';
+                    circle.style.left = `${docX}px`;
+                    circle.style.top = `${docY}px`;
+                    circle.style.borderRadius = '50%';
+                    circle.style.backgroundColor = '#026aa1'; // Blue color
+                    circle.style.width = '0px';
+                    circle.style.height = '0px';
+                    circle.style.transform = 'translate(-50%, -50%)'; // Center on (x, y)
+                    circle.style.pointerEvents = 'none';
+                    circle.style.zIndex = '9998'; // Below the pointer
+                    circle.style.opacity = '0.7'; // Initial opacity
+                    document.body.appendChild(circle);
+                    // Animate the circle
+                    const animation = circle.animate([
+                        { width: '0px', height: '0px', opacity: 0.7 }, // Start state
+                        { width: '50px', height: '50px', opacity: 0 } // End state
+                    ], {
+                        duration: 500, // 500ms duration
+                        easing: 'ease-out'
+                    });
+                    // Remove circle after animation
+                    animation.onfinish = () => {
+                        circle.remove();
+                    };
+                }
+                // --- Pointer Logic (Always runs) ---
+                // Check if the visual indicator already exists
+                let pointerElement = document.getElementById(id);
+                // If it doesn't exist, create it with all necessary styling
+                if (!pointerElement) {
+                    pointerElement = document.createElement('div');
+                    pointerElement.id = id;
+                    pointerElement.style.position = 'fixed'; // Use fixed positioning for viewport-relative
+                    pointerElement.style.width = '32px';
+                    pointerElement.style.height = '32px';
+                    pointerElement.style.zIndex = '2147483647'; // Max z-index
+                    pointerElement.style.pointerEvents = 'none'; // Don't interfere with actual clicks
+                    // Notice that transition is 300ms
+                    pointerElement.style.transition = 'left 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), top 0.3s cubic-bezier(0.25, 0.1, 0.25, 1)';
+                    // Create SVG using DOM methods to avoid Trusted Types issues
+                    const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
+                    svg.setAttribute('width', '32');
+                    svg.setAttribute('height', '32');
+                    svg.setAttribute('viewBox', '0 0 113.50408 99.837555');
+                    const g = document.createElementNS('http://www.w3.org/2000/svg', 'g');
+                    g.setAttribute('transform', 'translate(-413.10686,-501.19661)');
+                    const path1 = document.createElementNS('http://www.w3.org/2000/svg', 'path');
+                    path1.setAttribute('style', 'fill:#026aa1;fill-opacity:1;stroke:#000000;stroke-width:0');
+                    path1.setAttribute('d', 'm 416.1069,504.1966 52.47697,93.83813 8.33253,-57.61019 z');
+                    const path2 = document.createElementNS('http://www.w3.org/2000/svg', 'path');
+                    path2.setAttribute('style', 'fill:#0384c7;fill-opacity:1;stroke:#000000;stroke-width:0');
+                    path2.setAttribute('d', 'm 416.1069,504.1966 60.8095,36.22794 46.69517,-34.75524 z');
+                    const path3 = document.createElementNS('http://www.w3.org/2000/svg', 'path');
+                    path3.setAttribute('style', 'fill:#0384c7;fill-opacity:0;stroke:#000000;stroke-width:6;stroke-linecap:round;stroke-linejoin:round');
+                    path3.setAttribute('d', 'm 416.1069,504.19658 52.47698,93.83813 8.33252,-57.61019 46.69517,-34.75521 -107.50467,-1.47273');
+                    g.appendChild(path1);
+                    g.appendChild(path2);
+                    g.appendChild(path3);
+                    svg.appendChild(g);
+                    pointerElement.appendChild(svg);
+                    document.body.appendChild(pointerElement);
+                }
+                //pointerElement.style.display = 'none';
+                // Update position - use viewport coordinates for fixed positioning
+                // Set the top-left corner to (viewportX, viewportY) and then translate by (-1px, -3px)
+                // to align the pointer tip (approx. at 1.27, 4.17 within the SVG) with the click point.
+                pointerElement.style.left = `${viewportX}px`;
+                pointerElement.style.top = `${viewportY}px`;
+                pointerElement.style.transform = 'translate(-1px, -3px)';
+            }, { x, y, id: this.visualElementId, showClickEffect });
+        }
+        catch (error) {
+            // For example when:
+            // TypeError: Failed to set the 'innerHTML' property on 'Element': This document requires 'TrustedHTML' assignment.
+            logger.trace(`Failed to draw visual: ${error.message}`);
+        }
+    }
+    async hide() {
+        try {
+            await this.page.evaluate((id) => {
+                const element = document.getElementById(id);
+                if (element) {
+                    element.style.display = 'none';
+                }
+            }, this.visualElementId);
+        }
+        catch {
+            logger.trace(`Failed to hide pointer`);
+        }
+    }
+    async show() {
+        try {
+            await this.page.evaluate((id) => {
+                const element = document.getElementById(id);
+                if (element) {
+                    // Revert to the default display value (usually 'block' for a div)
+                    element.style.display = '';
+                }
+            }, this.visualElementId);
+        }
+        catch {
+            logger.trace(`Failed to show pointer`);
+        }
+    }
+}

package/dist/web/visualizer/index.d.ts

@@ -0,0 +1,22 @@
+import { BrowserContext, Page } from "playwright";
+export interface ActionVisualizerOptions {
+    showCursor?: boolean;
+    showHoverCircle?: boolean;
+    showClickRipple?: boolean;
+    showDragLine?: boolean;
+    showTypeEffects?: boolean;
+}
+export declare class ActionVisualizer {
+    private options;
+    private context;
+    private page;
+    private cursor;
+    private mouseEffects;
+    private typeEffects;
+    constructor(context: BrowserContext, options: ActionVisualizerOptions);
+    setup(): Promise<void>;
+    setActivePage(page: Page): Promise<void>;
+    moveVirtualCursor(x: number, y: number): Promise<void>;
+    hideAll(): Promise<void>;
+    showAll(): Promise<void>;
+}

package/dist/web/visualizer/index.js

@@ -0,0 +1,47 @@
+import { CursorVisual } from "./cursor";
+import { MouseEffectVisual } from "./mouseEffects";
+import { TypeEffectVisual } from "./typeEffects";
+export class ActionVisualizer {
+    options;
+    context;
+    page;
+    cursor;
+    mouseEffects;
+    typeEffects;
+    constructor(context, options) {
+        this.context = context;
+        this.options = {
+            showCursor: options.showCursor ?? true,
+            showHoverCircle: options.showHoverCircle ?? false,
+            showClickRipple: options.showClickRipple ?? true,
+            showDragLine: options.showDragLine ?? false,
+            showTypeEffects: options.showTypeEffects ?? true
+        };
+        this.cursor = new CursorVisual();
+        this.mouseEffects = new MouseEffectVisual(this.options);
+        this.typeEffects = new TypeEffectVisual();
+    }
+    async setup() {
+        // await this.mouseEffects.setContext(this.context);
+        // if (this.options.showTypeEffects) {
+        //     await this.typeEffects.setContext(this.context);
+        // }
+        //this.showAll();
+    }
+    async setActivePage(page) {
+        this.page = page;
+        await this.cursor.setActivePage(page);
+        await this.mouseEffects.setActivePage(page);
+        await this.typeEffects.setActivePage(page);
+    }
+    async moveVirtualCursor(x, y) {
+        // Takes like 300ms for smooth anim
+        await this.cursor.move(x, y);
+    }
+    async hideAll() {
+        await this.cursor.hide();
+    }
+    async showAll() {
+        await this.cursor.show();
+    }
+}

package/dist/web/visualizer/mouseEffects.d.ts

@@ -0,0 +1,18 @@
+import { Page } from "playwright";
+interface MouseEffectOptions {
+    baseOpacity?: number;
+    showHoverCircle?: boolean;
+    showClickRipple?: boolean;
+    showDragLine?: boolean;
+}
+export declare class MouseEffectVisual {
+    private baseOpacity;
+    private showHoverCircle;
+    private showClickRipple;
+    private showDragLine;
+    private page;
+    constructor(options?: MouseEffectOptions);
+    setActivePage(page: Page): Promise<void>;
+    setupOnPage(): Promise<void>;
+}
+export {};