@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/web/harness.js

@@ -0,0 +1,340 @@
+import { PageStabilityAnalyzer } from "./stability";
+import { parseTypeContent } from "./util";
+import { ActionVisualizer } from "./visualizer";
+import logger from "@/logger";
+import { TabManager } from "./tabs";
+import { DOMTransformer } from "./transformer";
+import { Image } from '@/memory/image';
+import EventEmitter from "eventemitter3";
+export class WebHarness {
+    /**
+     * Executes web actions on a page
+     * Not responsible for browser lifecycle
+     */
+    context;
+    options;
+    stability;
+    visualizer;
+    transformer;
+    tabs;
+    events = new EventEmitter();
+    constructor(context, options = {}) {
+        //this.page = page;
+        this.context = context;
+        this.options = options;
+        this.stability = new PageStabilityAnalyzer({ disableVisualStability: true });
+        this.visualizer = new ActionVisualizer(this.context, this.options.visuals ?? {});
+        this.transformer = new DOMTransformer();
+        this.tabs = new TabManager(context, {
+            switchOnActivity: options.switchTabsOnActivity ?? true
+        });
+        // this.context.on('page', (page: Page) => {
+        //     this.setActivePage(page);
+        //     //logger.info('ayo we got a new page');
+        // });
+        this.tabs.events.on('tabChanged', async (page) => {
+            await this.setActivePage(page);
+            // need to wait for page to load before evaluating a script
+            //page.on('load', () => { this.transformer.setActivePage(page); });
+            //console.log('tabs:', await this.tabs.getState())
+        }, this);
+    }
+    async setActivePage(page) {
+        logger.trace(`WebHarness active page: ${page.url()}`);
+        this.stability.setActivePage(page);
+        await this.visualizer.setActivePage(page);
+        this.transformer.setActivePage(page);
+        this.events.emit('activePageChanged', page);
+    }
+    async retrieveTabState() {
+        return this.tabs.retrieveState();
+    }
+    // setActivePage(page: Page) {
+    //     this.page = page;
+    //     this.stability.setActivePage(this.page);
+    //     this.visualizer.setActivePage(this.page);
+    // }
+    async start() {
+        // Initialize tab manager first
+        await this.tabs.initialize();
+        if (this.context.pages().length > 0) {
+            // If context already contains a page, set it as active
+            this.tabs.setActivePage(this.context.pages()[0]);
+        }
+        else {
+            const page = await this.context.newPage();
+            // Force the initial page to be set as active and emit tabChanged
+            this.tabs.setActivePage(page);
+        }
+        await this.visualizer.setup();
+    }
+    async stop() {
+        // Clean up tab manager resources
+        this.tabs.destroy();
+    }
+    get page() {
+        return this.tabs.getActivePage();
+    }
+    async screenshot(options = {}) {
+        /**
+         * Get b64 encoded string of screenshot (PNG) with screen dimensions
+         */
+        // Target page, context or browser has been closed
+        let dpr;
+        let buffer;
+        const retries = 3;
+        for (let attempt = 0; attempt <= retries; attempt++) {
+            try {
+                dpr = await this.page.evaluate(() => window.devicePixelRatio);
+                buffer = await this.page.screenshot({ type: 'png', ...options });
+                break; // Success! Exit the retry loop
+            }
+            catch (err) {
+                // A few possibilities:
+                // 1. Target page, context or browser has been closed
+                // 2. Page navigation in progress
+                // In theory 2 shouldn't shouldn't happen during typical execution as we wait for page load - unless screenshot is triggered at an usual time.
+                const error = err;
+                if (error.message.includes('Target page, context or browser has been closed')) {
+                    // Irrecoverable, no point in retrying
+                    throw new Error("Attempted to take screenshot but page, context or browser is closed");
+                }
+                if (attempt >= retries) {
+                    throw new Error(`Unable to capture screenshot after retries, error: ${error.message}`);
+                }
+            }
+        }
+        const base64data = buffer.toString('base64');
+        const image = Image.fromBase64(base64data);
+        // Now, need to rescale the image based on DPR. This is so that:
+        // (1) Save on tokens, dont need huge high res images
+        // (2) More importantly, clicks happen in the standard resolution space, so need to do this for coordinates to be correct
+        //     for any agent not using a virtual screen space (e.g. those that aren't Claude)
+        const { width, height } = await image.getDimensions();
+        const rescaledImage = await image.resize(width / dpr, height / dpr);
+        return rescaledImage;
+    }
+    // async goto(url: string) {
+    //     // No need to redraw here anymore, the 'load' event listener handles it
+    //     await this.page.goto(url);
+    // }
+    async _type(content) {
+        /** Util for typing + keypresses */
+        const chunks = parseTypeContent(content);
+        // Total typing period to make typing more natural, in ms
+        const totalTextDelay = 500;
+        let totalTextLength = 0;
+        for (const chunk of chunks) {
+            if (chunk != '<enter>' && chunk != '<tab>') {
+                totalTextLength += chunk.length;
+            }
+        }
+        for (const chunk of chunks) {
+            if (chunk == '<enter>') {
+                await this.page.keyboard.press('Enter');
+            }
+            else if (chunk == '<tab>') {
+                await this.page.keyboard.press('Tab');
+            }
+            else {
+                const chunkProportion = chunk.length / totalTextLength;
+                const chunkDelay = totalTextDelay * chunkProportion;
+                const chunkCharDelay = chunkDelay / chunk.length;
+                await this.page.keyboard.type(chunk, { delay: chunkCharDelay });
+            }
+        }
+    }
+    // safer might be Coordinate interface/obj tied to certain screen space dims
+    async transformCoordinates({ x, y }) {
+        const virtual = this.options.virtualScreenDimensions;
+        if (!virtual) {
+            return { x, y };
+        }
+        let vp = this.page.viewportSize();
+        if (!vp) {
+            vp = await this.page.evaluate(() => ({
+                width: window.innerWidth,
+                height: window.innerHeight
+            }));
+        }
+        if (!vp)
+            throw new Error("Could not get viewport dimensions to transform coordinates");
+        return {
+            x: x * (vp.width / virtual.width),
+            y: y * (vp.height / virtual.height),
+        };
+    }
+    async click({ x, y }, options) {
+        if (options?.transform ?? true)
+            ({ x, y } = await this.transformCoordinates({ x, y }));
+        // console.log("x:", x);
+        // console.log("y:", y);
+        //await this.visualizer.visualizeAction(x, y);
+        //await this.page.mouse.click(x, y);
+        //await this.page.mouse.move(x, y, { steps: 20 });
+        //console.log('clicking:', x, y);
+        // const loc = this.page.getByText('Where are you going?');
+        // console.log('found:', loc);
+        // await loc.click();
+        await this._click(x, y);
+        // await this.page.waitForTimeout(1000);
+        // await this.page.mouse.click(x, y);
+        // await this.page.waitForTimeout(1000);
+        // await this.page.mouse.click(x, y);
+        // await this.page.waitForTimeout(1000);
+        // await this.page.mouse.click(x, y);
+        // await this.page.waitForTimeout(1000);
+        // await this.page.mouse.click(x, y);
+        // await this.page.waitForTimeout(1000);
+        // await Promise.all([
+        //     this.page.mouse.move(x, y, { steps: 20 }),
+        //     this.visualizer.visualizeAction(x, y),
+        // ]);
+        // await this.page.mouse.down();
+        // await this.page.waitForTimeout(200);
+        // await this.page.mouse.up();
+        // await this.page.evaluate(({ x, y }) => {
+        //     // Find the topmost element at the given coordinates
+        //     const targetElement = document.elementFromPoint(x, y);
+        //     if (!targetElement) {
+        //         console.error('No element found at coordinates:', x, y);
+        //         return;
+        //     }
+        //     // Create and dispatch the events with properties that mimic a real click
+        //     const options = {
+        //         bubbles: true,
+        //         cancelable: true,
+        //         composed: true,
+        //         // We can't set isTrusted, the browser forces it to false
+        //     };
+        //     targetElement.dispatchEvent(new MouseEvent('mouseover', options));
+        //     targetElement.dispatchEvent(new MouseEvent('mousedown', options));
+        //     targetElement.dispatchEvent(new MouseEvent('mouseup', options));
+        //     targetElement.dispatchEvent(new MouseEvent('click', options));
+        // }, { x, y });
+        await this.waitForStability();
+        //await this.visualizer.removeActionVisuals();
+    }
+    async _click(x, y, options) {
+        await Promise.all([
+            this.visualizer.moveVirtualCursor(x, y),
+            this.page.mouse.move(x, y, { steps: 20 })
+        ]);
+        // await this.visualizer.moveVirtualCursor(x, y);
+        // await this.page.mouse.move(x, y, { steps: 20 });
+        await this.visualizer.hideAll(); // hide / show pointer because no-pointer is not always consistent and visualizer can block click
+        await this.page.mouse.click(x, y);
+        await this.visualizer.showAll();
+    }
+    async rightClick({ x, y }, options) {
+        if (options?.transform ?? true)
+            ({ x, y } = await this.transformCoordinates({ x, y }));
+        await this._click(x, y, { button: "right" });
+        await this.waitForStability();
+    }
+    async doubleClick({ x, y }, options) {
+        if (options?.transform ?? true)
+            ({ x, y } = await this.transformCoordinates({ x, y }));
+        await this.visualizer.moveVirtualCursor(x, y);
+        await this.visualizer.hideAll();
+        await this.page.mouse.dblclick(x, y);
+        await this.visualizer.showAll();
+        await this.waitForStability();
+    }
+    async drag({ x1, y1, x2, y2 }, options) {
+        if (options?.transform ?? true)
+            ({ x: x1, y: y1 } = await this.transformCoordinates({ x: x1, y: y1 }));
+        if (options?.transform ?? true)
+            ({ x: x2, y: y2 } = await this.transformCoordinates({ x: x2, y: y2 }));
+        //console.log(`Dragging: (${x1}, ${y1}) -> (${x2}, ${y2})`);
+        await this.page.mouse.move(x1, y1, { steps: 1 });
+        await this.page.mouse.down();
+        await this.visualizer.moveVirtualCursor(x1, y1);
+        await this.page.waitForTimeout(500);
+        await Promise.all([
+            this.page.mouse.move(x2, y2, { steps: 20 }),
+            this.visualizer.moveVirtualCursor(x2, y2)
+        ]);
+        // await this.page.mouse.move(x2, y2, { steps: 100 });
+        // await this.visualizer.visualizeAction(x2, y2);
+        await this.page.mouse.up();
+        await this.waitForStability();
+        //await this.visualizer.removeActionVisuals();
+    }
+    async type({ content }) {
+        await this._type(content);
+        await this.waitForStability();
+    }
+    async clickAndType({ x, y, content }, options) {
+        // TODO: transforms incorrect for moondream grounding with virtual screen dims (claude) - unsure why
+        //console.log(`Pre transform: ${x}, ${y}`);
+        if (options?.transform ?? true)
+            ({ x, y } = await this.transformCoordinates({ x, y }));
+        //console.log(`Post transform: ${x}, ${y}`);
+        await this.visualizer.moveVirtualCursor(x, y);
+        this._click(x, y);
+        await this._type(content);
+        await this.waitForStability();
+    }
+    async scroll({ x, y, deltaX, deltaY }, options) {
+        if (options?.transform ?? true)
+            ({ x, y } = await this.transformCoordinates({ x, y }));
+        await this.visualizer.moveVirtualCursor(x, y);
+        await this.page.mouse.move(x, y);
+        await this.page.mouse.wheel(deltaX, deltaY);
+        await this.waitForStability();
+    }
+    async switchTab({ index }) {
+        await this.tabs.switchTab(index);
+        await this.waitForStability();
+    }
+    async newTab() {
+        await this.context.newPage();
+        // Reasonable default and less confusing than white about:blank page
+        await this.navigate("https://google.com");
+    }
+    async navigate(url) {
+        // Only wait for DOM content on goto since we handle waiting for network idle etc ourselves
+        await this.page.goto(url, { waitUntil: 'domcontentloaded' });
+        await this.waitForStability();
+    }
+    async selectAll() {
+        await this.page.keyboard.down('ControlOrMeta');
+        await this.page.keyboard.press('KeyA');
+        await this.page.keyboard.up('ControlOrMeta');
+    }
+    async enter() {
+        await this.page.keyboard.press('Enter');
+    }
+    async backspace() {
+        await this.page.keyboard.press('Backspace');
+    }
+    async tab() {
+        await this.page.keyboard.press('Tab');
+    }
+    async goBack() {
+        await this.page.goBack();
+    }
+    async executeAction(action) {
+        if (action.variant === 'click') {
+            await this.click(action);
+        }
+        else if (action.variant === 'type') {
+            await this.clickAndType(action);
+        }
+        else if (action.variant === 'scroll') {
+            await this.scroll(action);
+        }
+        else if (action.variant === 'tab') {
+            await this.switchTab(action);
+        }
+        else {
+            throw Error(`Unhandled web action variant: ${action.variant}`);
+        }
+        //await this.stability.waitForStability();
+        //await this.visualizer.redrawLastPosition();
+    }
+    async waitForStability(timeout) {
+        await this.stability.waitForStability(timeout);
+    }
+}

package/dist/web/stability.d.ts

@@ -0,0 +1,57 @@
+import { Page } from 'playwright';
+export interface ImageDiff {
+    /** Difference score between 0-1, where 0 means identical */
+    difference: number;
+    /** Optional error message if comparison failed */
+    error?: string;
+}
+export interface PageStabilityOptions {
+    /** Threshold below which screenshots are considered similar (0-1) */
+    differenceThreshold?: number;
+    /** Number of consecutive stable checks required to confirm stability */
+    requiredStableChecks?: number;
+    /** Interval between stability checks in ms */
+    checkInterval?: number;
+    /** Whether to log detailed stability information */
+    /** Minimum time to wait before getting page state */
+    minimumWaitPageLoadTime?: number;
+    /** Time to wait for network requests to finish before getting page state */
+    waitForNetworkIdleTime?: number;
+    /** Maximum time to wait for page load before proceeding anyway */
+    maximumWaitPageLoadTime?: number;
+    /** Whether to disable visual stability checking */
+    disableVisualStability?: boolean;
+}
+/**
+ * Utility class to check for visual stability of a page
+ */
+export declare class PageStabilityAnalyzer {
+    private page;
+    private options;
+    private lastStart;
+    private logger;
+    constructor(options?: PageStabilityOptions);
+    setActivePage(page: Page): void;
+    private log;
+    /**
+     * Compare two screenshots and return their difference score
+     * @returns Difference as float between 0-1, where 0 means identical
+     */
+    private compareScreenshots;
+    /**
+     * Wait for the network to become stable
+     * This monitors network requests and waits until there's a period of inactivity
+     * @param timeout Maximum time to wait for network stability
+     */
+    waitForNetworkStability(timeout?: number): Promise<void>;
+    /**
+     * Wait for the page to become visually stable
+     * @param timeout Maximum time to wait for stability in ms
+     */
+    waitForVisualStability(timeout?: number): Promise<void>;
+    /**
+     * Wait for both network and visual stability
+     * @param timeout Maximum time to wait for page load
+     */
+    waitForStability(timeout?: number): Promise<void>;
+}