@ddwang/magnitude-core 0.3.1-ddwang.2 → 0.3.1-ddwang.21

package/dist/web/harness.js

@@ -1,401 +0,0 @@
-import { PageStabilityAnalyzer } from "./stability";
-import { parseTypeContent } from "./util";
-import { ActionVisualizer } from "./visualizer";
-import logger from "@/logger";
-import { TabManager } from "./tabs";
-import { DOMTransformer } from "./transformer";
-import { Image } from '@/memory/image';
-import EventEmitter from "eventemitter3";
-import { checkOperation, drainAll, operationSleep } from '@/common/operation';
-export class WebHarness {
-    /**
-     * Executes web actions on a page
-     * Not responsible for browser lifecycle
-     */
-    context;
-    options;
-    stability;
-    visualizer;
-    transformer;
-    tabs;
-    events = new EventEmitter();
-    constructor(context, options = {}) {
-        //this.page = page;
-        this.context = context;
-        this.options = options;
-        this.stability = new PageStabilityAnalyzer({ disableVisualStability: true });
-        this.visualizer = new ActionVisualizer(this.context, this.options.visuals ?? {});
-        this.transformer = new DOMTransformer();
-        this.tabs = new TabManager(context, {
-            switchOnActivity: options.switchTabsOnActivity ?? true
-        });
-        // this.context.on('page', (page: Page) => {
-        //     this.setActivePage(page);
-        //     //logger.info('ayo we got a new page');
-        // });
-        this.tabs.events.on('tabChanged', async (page) => {
-            await this.setActivePage(page);
-            // need to wait for page to load before evaluating a script
-            //page.on('load', () => { this.transformer.setActivePage(page); });
-            //console.log('tabs:', await this.tabs.getState())
-        }, this);
-    }
-    async setActivePage(page) {
-        logger.trace(`WebHarness active page: ${page.url()}`);
-        this.stability.setActivePage(page);
-        await this.visualizer.setActivePage(page);
-        this.transformer.setActivePage(page);
-        this.events.emit('activePageChanged', page);
-    }
-    async retrieveTabState() {
-        return this.tabs.retrieveState();
-    }
-    // setActivePage(page: Page) {
-    //     this.page = page;
-    //     this.stability.setActivePage(this.page);
-    //     this.visualizer.setActivePage(this.page);
-    // }
-    async start() {
-        // Initialize tab manager first
-        await this.tabs.initialize();
-        if (this.context.pages().length > 0) {
-            // If context already contains a page, set it as active
-            this.tabs.setActivePage(this.context.pages()[0]);
-        }
-        else {
-            const page = await this.context.newPage();
-            // Force the initial page to be set as active and emit tabChanged
-            this.tabs.setActivePage(page);
-        }
-        await this.visualizer.setup();
-    }
-    async stop() {
-        // Clean up tab manager resources
-        this.tabs.destroy();
-    }
-    get page() {
-        return this.tabs.getActivePage();
-    }
-    async screenshot(options = {}) {
-        /**
-         * Get b64 encoded string of screenshot (PNG) with screen dimensions
-         */
-        // Target page, context or browser has been closed
-        let dpr;
-        let buffer;
-        const retries = 3;
-        for (let attempt = 0; attempt <= retries; attempt++) {
-            checkOperation();
-            try {
-                dpr = await this.page.evaluate(() => window.devicePixelRatio);
-                checkOperation();
-                buffer = await this.page.screenshot({ type: 'png', ...options });
-                checkOperation();
-                break; // Success! Exit the retry loop
-            }
-            catch (err) {
-                checkOperation();
-                // A few possibilities:
-                // 1. Target page, context or browser has been closed
-                // 2. Page navigation in progress
-                // In theory 2 shouldn't shouldn't happen during typical execution as we wait for page load - unless screenshot is triggered at an usual time.
-                const error = err;
-                if (error.message.includes('Target page, context or browser has been closed')) {
-                    // Irrecoverable, no point in retrying
-                    throw new Error("Attempted to take screenshot but page, context or browser is closed");
-                }
-                if (attempt >= retries) {
-                    throw new Error(`Unable to capture screenshot after retries, error: ${error.message}`);
-                }
-            }
-        }
-        const base64data = buffer.toString('base64');
-        const image = Image.fromBase64(base64data);
-        // Now, need to rescale the image based on DPR. This is so that:
-        // (1) Save on tokens, dont need huge high res images
-        // (2) More importantly, clicks happen in the standard resolution space, so need to do this for coordinates to be correct
-        //     for any agent not using a virtual screen space (e.g. those that aren't Claude)
-        const { width, height } = await image.getDimensions();
-        const rescaledImage = await image.resize(width / dpr, height / dpr);
-        return rescaledImage;
-    }
-    // async goto(url: string) {
-    //     // No need to redraw here anymore, the 'load' event listener handles it
-    //     await this.page.goto(url);
-    // }
-    async _type(content) {
-        /** Util for typing + keypresses */
-        const chunks = parseTypeContent(content);
-        // Total typing period to make typing more natural, in ms
-        const totalTextDelay = 500;
-        let totalTextLength = 0;
-        for (const chunk of chunks) {
-            if (chunk != '<enter>' && chunk != '<tab>') {
-                totalTextLength += chunk.length;
-            }
-        }
-        for (const chunk of chunks) {
-            checkOperation();
-            if (chunk == '<enter>') {
-                await this.page.keyboard.press('Enter');
-            }
-            else if (chunk == '<tab>') {
-                await this.page.keyboard.press('Tab');
-            }
-            else {
-                const chunkProportion = chunk.length / totalTextLength;
-                const chunkDelay = totalTextDelay * chunkProportion;
-                const chunkCharDelay = chunkDelay / chunk.length;
-                await this.page.keyboard.type(chunk, { delay: chunkCharDelay });
-            }
-        }
-    }
-    // safer might be Coordinate interface/obj tied to certain screen space dims
-    async transformCoordinates({ x, y }) {
-        const virtual = this.options.virtualScreenDimensions;
-        if (!virtual) {
-            return { x, y };
-        }
-        let vp = this.page.viewportSize();
-        if (!vp) {
-            vp = await this.page.evaluate(() => ({
-                width: window.innerWidth,
-                height: window.innerHeight
-            }));
-        }
-        if (!vp)
-            throw new Error("Could not get viewport dimensions to transform coordinates");
-        return {
-            x: x * (vp.width / virtual.width),
-            y: y * (vp.height / virtual.height),
-        };
-    }
-    async click({ x, y }, options) {
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        // console.log("x:", x);
-        // console.log("y:", y);
-        //await this.visualizer.visualizeAction(x, y);
-        //await this.page.mouse.click(x, y);
-        //await this.page.mouse.move(x, y, { steps: 20 });
-        //console.log('clicking:', x, y);
-        // const loc = this.page.getByText('Where are you going?');
-        // console.log('found:', loc);
-        // await loc.click();
-        await this._click(x, y);
-        // await this.page.waitForTimeout(1000);
-        // await this.page.mouse.click(x, y);
-        // await this.page.waitForTimeout(1000);
-        // await this.page.mouse.click(x, y);
-        // await this.page.waitForTimeout(1000);
-        // await this.page.mouse.click(x, y);
-        // await this.page.waitForTimeout(1000);
-        // await this.page.mouse.click(x, y);
-        // await this.page.waitForTimeout(1000);
-        // await Promise.all([
-        //     this.page.mouse.move(x, y, { steps: 20 }),
-        //     this.visualizer.visualizeAction(x, y),
-        // ]);
-        // await this.page.mouse.down();
-        // await this.page.waitForTimeout(200);
-        // await this.page.mouse.up();
-        // await this.page.evaluate(({ x, y }) => {
-        //     // Find the topmost element at the given coordinates
-        //     const targetElement = document.elementFromPoint(x, y);
-        //     if (!targetElement) {
-        //         console.error('No element found at coordinates:', x, y);
-        //         return;
-        //     }
-        //     // Create and dispatch the events with properties that mimic a real click
-        //     const options = {
-        //         bubbles: true,
-        //         cancelable: true,
-        //         composed: true,
-        //         // We can't set isTrusted, the browser forces it to false
-        //     };
-        //     targetElement.dispatchEvent(new MouseEvent('mouseover', options));
-        //     targetElement.dispatchEvent(new MouseEvent('mousedown', options));
-        //     targetElement.dispatchEvent(new MouseEvent('mouseup', options));
-        //     targetElement.dispatchEvent(new MouseEvent('click', options));
-        // }, { x, y });
-        await this.waitForStability();
-        //await this.visualizer.removeActionVisuals();
-    }
-    async _click(x, y, options) {
-        checkOperation();
-        await drainAll([
-            this.visualizer.moveVirtualCursor(x, y),
-            this.page.mouse.move(x, y, { steps: 20 })
-        ]);
-        // await this.visualizer.moveVirtualCursor(x, y);
-        // await this.page.mouse.move(x, y, { steps: 20 });
-        checkOperation();
-        await this.visualizer.hideAll(); // The visualizer can block clicks.
-        try {
-            checkOperation();
-            await this.page.mouse.click(x, y);
-        }
-        finally {
-            await this.visualizer.showAll();
-        }
-    }
-    async hover({ x, y }, options) {
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        checkOperation();
-        await drainAll([
-            this.visualizer.moveVirtualCursor(x, y),
-            this.page.mouse.move(x, y, { steps: 20 })
-        ]);
-        await this.waitForStability();
-    }
-    async rightClick({ x, y }, options) {
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        await this._click(x, y, { button: "right" });
-        await this.waitForStability();
-    }
-    async doubleClick({ x, y }, options) {
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        checkOperation();
-        await this.visualizer.moveVirtualCursor(x, y);
-        checkOperation();
-        await this.visualizer.hideAll();
-        try {
-            checkOperation();
-            await this.page.mouse.dblclick(x, y);
-        }
-        finally {
-            await this.visualizer.showAll();
-        }
-        await this.waitForStability();
-    }
-    async drag({ x1, y1, x2, y2 }, options) {
-        if (options?.transform ?? true)
-            ({ x: x1, y: y1 } = await this.transformCoordinates({ x: x1, y: y1 }));
-        if (options?.transform ?? true)
-            ({ x: x2, y: y2 } = await this.transformCoordinates({ x: x2, y: y2 }));
-        //console.log(`Dragging: (${x1}, ${y1}) -> (${x2}, ${y2})`);
-        checkOperation();
-        await this.page.mouse.move(x1, y1, { steps: 1 });
-        checkOperation();
-        await this.page.mouse.down();
-        try {
-            checkOperation();
-            await this.visualizer.moveVirtualCursor(x1, y1);
-            await operationSleep(500);
-            checkOperation();
-            await drainAll([
-                this.page.mouse.move(x2, y2, { steps: 20 }),
-                this.visualizer.moveVirtualCursor(x2, y2)
-            ]);
-        }
-        finally {
-            // Release input state even on cancellation; an already-sent drag cannot be undone.
-            await this.page.mouse.up();
-        }
-        await this.waitForStability();
-        //await this.visualizer.removeActionVisuals();
-    }
-    async type({ content }) {
-        await this._type(content);
-        await this.waitForStability();
-    }
-    async clickAndType({ x, y, content }, options) {
-        // TODO: transforms incorrect for moondream grounding with virtual screen dims (claude) - unsure why
-        //console.log(`Pre transform: ${x}, ${y}`);
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        //console.log(`Post transform: ${x}, ${y}`);
-        await this._click(x, y);
-        await this._type(content);
-        await this.waitForStability();
-    }
-    async scroll({ x, y, deltaX, deltaY }, options) {
-        if (options?.transform ?? true)
-            ({ x, y } = await this.transformCoordinates({ x, y }));
-        checkOperation();
-        await this.visualizer.moveVirtualCursor(x, y);
-        checkOperation();
-        await this.page.mouse.move(x, y);
-        checkOperation();
-        await this.page.mouse.wheel(deltaX, deltaY);
-        await this.waitForStability();
-    }
-    async switchTab({ index }) {
-        checkOperation();
-        await this.tabs.switchTab(index);
-        await this.waitForStability();
-    }
-    async newTab() {
-        checkOperation();
-        await this.context.newPage();
-        // Reasonable default and less confusing than white about:blank page
-        await this.navigate("https://google.com");
-    }
-    async navigate(url) {
-        checkOperation();
-        // Only wait for DOM content on goto since we handle waiting for network idle etc ourselves
-        await this.page.goto(url, { waitUntil: 'domcontentloaded' });
-        await this.waitForStability();
-    }
-    async selectAll() {
-        checkOperation();
-        await this.page.keyboard.down('ControlOrMeta');
-        try {
-            checkOperation();
-            await this.page.keyboard.press('KeyA');
-        }
-        finally {
-            await this.page.keyboard.up('ControlOrMeta');
-        }
-    }
-    async enter() {
-        checkOperation();
-        await this.page.keyboard.press('Enter');
-    }
-    async backspace() {
-        checkOperation();
-        await this.page.keyboard.press('Backspace');
-    }
-    async tab() {
-        checkOperation();
-        await this.page.keyboard.press('Tab');
-    }
-    async goBack() {
-        checkOperation();
-        await this.page.goBack();
-    }
-    async escape() {
-        checkOperation();
-        await this.page.keyboard.press('Escape');
-    }
-    async executeAction(action) {
-        if (action.variant === 'click') {
-            await this.click(action);
-        }
-        else if (action.variant === 'hover') {
-            await this.hover(action);
-        }
-        else if (action.variant === 'type') {
-            await this.clickAndType(action);
-        }
-        else if (action.variant === 'scroll') {
-            await this.scroll(action);
-        }
-        else if (action.variant === 'tab') {
-            await this.switchTab(action);
-        }
-        else {
-            throw Error(`Unhandled web action variant: ${action.variant}`);
-        }
-        //await this.stability.waitForStability();
-        //await this.visualizer.redrawLastPosition();
-    }
-    async waitForStability(timeout) {
-        checkOperation();
-        await this.stability.waitForStability(timeout);
-        checkOperation();
-    }
-}

package/dist/web/recovery.d.ts

@@ -1,46 +0,0 @@
-import type { Action } from '@/actions/types';
-export type BlockReason = 'rate_limit' | 'subscription' | 'authentication' | 'no_progress';
-export interface BrowserBlock {
-    reason: BlockReason;
-    evidence: string;
-    retryAt?: number;
-}
-export declare class BrowserBlockedError extends Error {
-    readonly block: BrowserBlock;
-    constructor(block: BrowserBlock);
-}
-export interface HttpDiagnostic {
-    timestamp: number;
-    url: string;
-    status: number;
-    retryAt?: number;
-    navigation?: boolean;
-}
-export declare function diagnosticUrl(value: string): string;
-export declare function retryAt(value: string | undefined, now: number): number | undefined;
-export declare function detectBlock(headings: string[], response?: HttpDiagnostic): BrowserBlock | undefined;
-export interface RecoveryOptions {
-    maxRateLimitWaitMs?: number;
-    repeatedActionLimit?: number;
-    /** Opt in to heuristic loop warnings and automatic no-progress termination. */
-    noProgress?: boolean;
-}
-export declare class BrowserRecovery {
-    block?: BrowserBlock;
-    warning?: string;
-    waitUntil?: number;
-    private rateWaitMs;
-    private blockedActions;
-    private recent;
-    private recentStates;
-    private repetitions;
-    private lastFingerprint?;
-    readonly maxRateLimitWaitMs: number;
-    readonly repeatedActionLimit: number;
-    readonly noProgress: boolean;
-    constructor(options?: RecoveryOptions);
-    reset(): void;
-    observe(fingerprint: string, action: Action | undefined, block: BrowserBlock | undefined, now?: number): void;
-    check(): void;
-    waitDuration(requestedMs: number, now?: number, deadline?: number): number;
-}

package/dist/web/recovery.js

@@ -1,137 +0,0 @@
-export class BrowserBlockedError extends Error {
-    block;
-    constructor(block) {
-        super(`Browser blocked (${block.reason}): ${block.evidence}`);
-        this.block = block;
-        this.name = 'BrowserBlockedError';
-    }
-}
-// Strip userinfo, query strings and fragments; do not persist arbitrary headers.
-export function diagnosticUrl(value) {
-    try {
-        const url = new URL(value);
-        return `${url.origin}${url.pathname}`;
-    }
-    catch {
-        return 'unknown';
-    }
-}
-export function retryAt(value, now) {
-    if (!value)
-        return undefined;
-    const seconds = Number(value);
-    if (Number.isFinite(seconds))
-        return seconds >= 0 ? now + seconds * 1000 : undefined;
-    const date = Date.parse(value);
-    return Number.isFinite(date) ? Math.max(now, date) : undefined;
-}
-export function detectBlock(headings, response) {
-    const rate = headings.find(text => /too many requests|you have exceeded a secondary rate limit|rate limit exceeded/i.test(text));
-    if ((response?.status === 429 && response.navigation !== false) || rate)
-        return {
-            reason: 'rate_limit', evidence: rate ?? `HTTP 429 at ${response.url}`,
-            retryAt: response?.retryAt ?? (response ? response.timestamp + 60_000 : undefined),
-        };
-    const subscription = headings.find(text => /subscri(?:be|ption required).{0,60}(?:to continue|to (?:read|access)|required)/i.test(text));
-    if (subscription)
-        return { reason: 'subscription', evidence: subscription };
-    const authentication = headings.find(text => /(?:sign|log) in to (?:continue|read|access)/i.test(text));
-    if (authentication)
-        return { reason: 'authentication', evidence: authentication };
-    // A bare 403 can have many causes; it is not proof of rate limiting.
-}
-export class BrowserRecovery {
-    block;
-    warning;
-    waitUntil;
-    rateWaitMs = 0;
-    blockedActions = 0;
-    recent = [];
-    recentStates = new Set();
-    repetitions = 0;
-    lastFingerprint;
-    maxRateLimitWaitMs;
-    repeatedActionLimit;
-    noProgress;
-    constructor(options = {}) {
-        this.maxRateLimitWaitMs = options.maxRateLimitWaitMs ?? 120_000;
-        this.repeatedActionLimit = options.repeatedActionLimit ?? 6;
-        this.noProgress = options.noProgress ?? false;
-        if (!Number.isFinite(this.maxRateLimitWaitMs) || this.maxRateLimitWaitMs < 0
-            || !Number.isSafeInteger(this.repeatedActionLimit) || this.repeatedActionLimit < 3) {
-            throw new Error('Invalid browser recovery limits');
-        }
-    }
-    reset() {
-        this.block = undefined;
-        this.warning = undefined;
-        this.rateWaitMs = 0;
-        this.blockedActions = 0;
-        this.recent = [];
-        this.recentStates.clear();
-        this.repetitions = 0;
-        this.lastFingerprint = undefined;
-    }
-    observe(fingerprint, action, block, now = Date.now()) {
-        const previousFingerprint = this.lastFingerprint ?? fingerprint;
-        if (block?.reason !== this.block?.reason || fingerprint !== this.lastFingerprint)
-            this.blockedActions = 0;
-        if (fingerprint !== this.lastFingerprint || block) {
-            this.warning = undefined;
-            this.repetitions = 0;
-        }
-        this.lastFingerprint = fingerprint;
-        // New evidence breaks a loop, even when reaching it requires familiar paths.
-        // Keep a bounded LRU of states separate from the repetition history.
-        if (!this.recentStates.has(fingerprint) || block)
-            this.recent = [];
-        this.recentStates.delete(fingerprint);
-        this.recentStates.add(fingerprint);
-        if (this.recentStates.size > 30)
-            this.recentStates.delete(this.recentStates.values().next().value);
-        if (block?.reason === 'rate_limit') {
-            block.retryAt ??= this.block?.reason === 'rate_limit' ? this.block.retryAt : now + 60_000;
-        }
-        this.block = block;
-        // Waiting is deliberate inactivity, not evidence of a navigation loop.
-        if (!action || action.variant === 'wait' || action.variant === 'mouse:hover')
-            return;
-        if (block) {
-            this.blockedActions++;
-            return;
-        }
-        if (!this.noProgress)
-            return;
-        // Returning from different pages is not repeating the same transition.
-        // Ignore coordinates so jitter cannot disguise genuinely unchanged clicks.
-        const key = JSON.stringify([previousFingerprint, action.variant, fingerprint]);
-        this.recent.push(key);
-        if (this.recent.length > 30)
-            this.recent.shift();
-        this.repetitions = this.recent.filter(value => value === key).length;
-        this.warning = this.repetitions >= 3
-            ? 'This action has repeatedly produced a previously seen page state. Review tried searches and visited pages; use a materially different approach, or report browser:blocked with reason no_progress. Do not repeat the same search or click without new evidence.'
-            : undefined;
-    }
-    check() {
-        if (this.block && this.block.reason !== 'rate_limit' && this.blockedActions >= 3) {
-            throw new BrowserBlockedError(this.block);
-        }
-        if (this.noProgress && !this.block && this.repetitions >= this.repeatedActionLimit) {
-            throw new BrowserBlockedError({ reason: 'no_progress', evidence: this.warning });
-        }
-    }
-    waitDuration(requestedMs, now = Date.now(), deadline) {
-        const duration = Math.max(requestedMs, (this.block?.retryAt ?? now) - now, 0);
-        if (this.block?.reason === 'rate_limit') {
-            if (deadline !== undefined && (this.block.retryAt ?? now) >= deadline) {
-                throw new BrowserBlockedError({ ...this.block, evidence: `${this.block.evidence}; Retry-After exceeds the remaining operation deadline` });
-            }
-            if (this.rateWaitMs + duration > this.maxRateLimitWaitMs) {
-                throw new BrowserBlockedError({ ...this.block, evidence: `${this.block.evidence}; required wait exceeds the remaining rate-limit wait budget` });
-            }
-            this.rateWaitMs += duration;
-        }
-        return duration;
-    }
-}

package/dist/web/recovery.test.d.ts

@@ -1 +0,0 @@
-export {};