@ddwang/magnitude-core 0.3.1-ddwang.2 → 0.3.1-ddwang.3

package/dist/memory/image.d.ts

@@ -1,4 +1,4 @@
-import { Image as BamlImage } from '@boundaryml/baml';
+import { type Image as BamlImage } from '@boundaryml/baml';
 import { StoredMedia } from './serde';
 import { Sharp } from 'sharp';
 import sharp from 'sharp';

package/dist/memory/image.js

@@ -1,4 +1,5 @@
-import { Image as BamlImage } from '@boundaryml/baml';
+// Default import avoids Node 18/22's incomplete CJS named-export detection for Image.
+import baml from '@boundaryml/baml';
 import sharp from 'sharp';
 export class Image {
     /**
@@ -60,7 +61,7 @@ export class Image {
     }
     async toBaml() {
         const { format, base64 } = await this.toJson();
-        return BamlImage.fromBase64(`image/${format}`, base64);
+        return baml.Image.fromBase64(`image/${format}`, base64);
     }
     async saveToFile(filepath) {
         await this.img.clone().toFile(filepath);

package/dist/memory/rendering/renderJsonParts.js

@@ -1,5 +1,5 @@
 import { Image } from "@/memory/image";
-import { Image as BamlImage } from '@boundaryml/baml';
+import baml from '@boundaryml/baml';
 async function buildJsonPartsRecursive(data, partsList, indent, currentLevel = 0, isRoot = false) {
     if (data instanceof Image) {
         const bamlImg = await data.toBaml();
@@ -8,7 +8,7 @@ async function buildJsonPartsRecursive(data, partsList, indent, currentLevel = 0
         }
         return;
     }
-    if (data instanceof BamlImage) {
+    if (data instanceof baml.Image) {
         partsList.push(data);
         return;
     }

package/dist/memory/rendering/renderXmlParts.d.ts

@@ -1,7 +1,7 @@
 /**
  * Convert observations to form that can be passed as BAML context, rendered as a custom XML-like structure.
  */
-import { Image as BamlImage } from '@boundaryml/baml';
+import { type Image as BamlImage } from '@boundaryml/baml';
 import { RenderableContent } from '@/memory/observation';
 export type MultiMediaContentPart = BamlImage | string;
 export declare function renderXmlParts(data: RenderableContent): Promise<MultiMediaContentPart[]>;

package/dist/memory/rendering/renderXmlParts.js

@@ -1,7 +1,7 @@
 /**
  * Convert observations to form that can be passed as BAML context, rendered as a custom XML-like structure.
  */
-import { Image as BamlImage } from '@boundaryml/baml';
+import baml from '@boundaryml/baml';
 import { Image } from '@/memory/image';
 //export type BamlRenderable = BamlImage | string;
 // export type BamlRenderable = {
@@ -17,7 +17,7 @@ async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList
         }
         return;
     }
-    if (data instanceof BamlImage) {
+    if (data instanceof baml.Image) {
         partsList.push(data);
         return;
     }
@@ -66,7 +66,7 @@ async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList
             if (currentStr)
                 mergedValueParts.push(currentStr);
             // Apply styling rules
-            if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof BamlImage) {
+            if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof baml.Image) {
                 partsList.push(`${indent}<${tagName}>`);
                 partsList.push(mergedValueParts[0]); // The BamlImage
                 partsList.push(`</${tagName}>`);

package/dist/web/harness.js

@@ -6,7 +6,7 @@ import { TabManager } from "./tabs";
 import { DOMTransformer } from "./transformer";
 import { Image } from '@/memory/image';
 import EventEmitter from "eventemitter3";
-import { checkOperation, drainAll, operationSleep } from '@/common/operation';
+import { checkOperation, drainAll, measureOperation, operationSleep } from '@/common/operation';
 export class WebHarness {
     /**
      * Executes web actions on a page
@@ -77,47 +77,35 @@ export class WebHarness {
         return this.tabs.getActivePage();
     }
     async screenshot(options = {}) {
-        /**
-         * Get b64 encoded string of screenshot (PNG) with screen dimensions
-         */
-        // Target page, context or browser has been closed
-        let dpr;
-        let buffer;
-        const retries = 3;
-        for (let attempt = 0; attempt <= retries; attempt++) {
-            checkOperation();
-            try {
-                dpr = await this.page.evaluate(() => window.devicePixelRatio);
-                checkOperation();
-                buffer = await this.page.screenshot({ type: 'png', ...options });
-                checkOperation();
-                break; // Success! Exit the retry loop
-            }
-            catch (err) {
+        return measureOperation('screenshot', async () => {
+            let dpr;
+            let buffer;
+            const retries = 3;
+            for (let attempt = 0; attempt <= retries; attempt++) {
                 checkOperation();
-                // A few possibilities:
-                // 1. Target page, context or browser has been closed
-                // 2. Page navigation in progress
-                // In theory 2 shouldn't shouldn't happen during typical execution as we wait for page load - unless screenshot is triggered at an usual time.
-                const error = err;
-                if (error.message.includes('Target page, context or browser has been closed')) {
-                    // Irrecoverable, no point in retrying
-                    throw new Error("Attempted to take screenshot but page, context or browser is closed");
+                try {
+                    dpr = await this.page.evaluate(() => window.devicePixelRatio);
+                    checkOperation();
+                    buffer = await this.page.screenshot({ type: 'png', ...options });
+                    checkOperation();
+                    break;
                 }
-                if (attempt >= retries) {
-                    throw new Error(`Unable to capture screenshot after retries, error: ${error.message}`);
+                catch (err) {
+                    checkOperation();
+                    const error = err;
+                    if (error.message.includes('Target page, context or browser has been closed')) {
+                        throw new Error("Attempted to take screenshot but page, context or browser is closed");
+                    }
+                    if (attempt >= retries) {
+                        throw new Error(`Unable to capture screenshot after retries, error: ${error.message}`);
+                    }
                 }
             }
-        }
-        const base64data = buffer.toString('base64');
-        const image = Image.fromBase64(base64data);
-        // Now, need to rescale the image based on DPR. This is so that:
-        // (1) Save on tokens, dont need huge high res images
-        // (2) More importantly, clicks happen in the standard resolution space, so need to do this for coordinates to be correct
-        //     for any agent not using a virtual screen space (e.g. those that aren't Claude)
-        const { width, height } = await image.getDimensions();
-        const rescaledImage = await image.resize(width / dpr, height / dpr);
-        return rescaledImage;
+            const image = Image.fromBase64(buffer.toString('base64'));
+            // Match browser coordinate space while avoiding high-DPR image tokens.
+            const { width, height } = await image.getDimensions();
+            return await image.resize(width / dpr, height / dpr);
+        });
     }
     // async goto(url: string) {
     //     // No need to redraw here anymore, the 'load' event listener handles it
@@ -395,7 +383,7 @@ export class WebHarness {
     }
     async waitForStability(timeout) {
         checkOperation();
-        await this.stability.waitForStability(timeout);
+        await measureOperation('stability', () => this.stability.waitForStability(timeout));
         checkOperation();
     }
 }

package/dist/web/recovery.d.ts

@@ -1,4 +1,5 @@
 import type { Action } from '@/actions/types';
+import type { OperationDiagnostics } from '@/common/operation';
 export type BlockReason = 'rate_limit' | 'subscription' | 'authentication' | 'no_progress';
 export interface BrowserBlock {
     reason: BlockReason;
@@ -7,6 +8,7 @@ export interface BrowserBlock {
 }
 export declare class BrowserBlockedError extends Error {
     readonly block: BrowserBlock;
+    readonly operation?: OperationDiagnostics;
     constructor(block: BrowserBlock);
 }
 export interface HttpDiagnostic {

package/dist/web/visualizer/cursor.d.ts

@@ -1,12 +1,13 @@
 import { Page } from "playwright";
 export declare class CursorVisual {
+    private animate;
     /**
      * Manages the visual indicator for actions on a page
      */
     private page;
     private visualElementId;
     private lastPosition;
-    constructor();
+    constructor(animate?: boolean);
     setActivePage(page: Page): Promise<void>;
     move(x: number, y: number): Promise<void>;
     setupOnPage(): Promise<void>;

package/dist/web/visualizer/cursor.js

@@ -1,15 +1,16 @@
 import { retryOnErrorIsSuccess } from "@/common";
 import logger from "@/logger";
-import { operationSleep } from '@/common/operation';
+import { measureOperation, operationSleep } from '@/common/operation';
 export class CursorVisual {
+    animate;
     /**
      * Manages the visual indicator for actions on a page
      */
     page;
     visualElementId = 'action-visual-indicator';
     lastPosition = null;
-    constructor() {
-        //this.page = page;
+    constructor(animate = true) {
+        this.animate = animate;
     }
     async setActivePage(page) {
         this.page = page;
@@ -19,13 +20,12 @@ export class CursorVisual {
         await retryOnErrorIsSuccess(this.setupOnPage.bind(this), { mode: 'retry_all', delayMs: 200, retryLimit: 5 });
     }
     async move(x, y) {
-        // Store the position
-        this.lastPosition = { x, y };
-        // Create or update the mouse pointer visual, showing the click effect
-        await this._drawVisual(x, y, false);
-        // The pointer visual takes 0.3s on the transition, but awaiting script evaluation does not wait for this to complete.
-        // So we wait 300ms manually.
-        await operationSleep(300);
+        await measureOperation('cursor', async () => {
+            this.lastPosition = { x, y };
+            await this._drawVisual(x, y, false);
+            if (this.animate)
+                await operationSleep(300);
+        });
     }
     async setupOnPage() {
         if (this.lastPosition) {
@@ -36,7 +36,7 @@ export class CursorVisual {
     // Internal method to handle the actual drawing logic
     async _drawVisual(x, y, showClickEffect) {
         try {
-            await this.page.evaluate(({ x, y, id, showClickEffect }) => {
+            await this.page.evaluate(({ x, y, id, showClickEffect, animate }) => {
                 // Use viewport coordinates directly (no scroll adjustment for fixed positioning)
                 const viewportX = x;
                 const viewportY = y;
@@ -83,8 +83,6 @@ export class CursorVisual {
                     pointerElement.style.height = '32px';
                     pointerElement.style.zIndex = '2147483647'; // Max z-index
                     pointerElement.style.pointerEvents = 'none'; // Don't interfere with actual clicks
-                    // Notice that transition is 300ms
-                    pointerElement.style.transition = 'left 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), top 0.3s cubic-bezier(0.25, 0.1, 0.25, 1)';
                     // Create SVG using DOM methods to avoid Trusted Types issues
                     const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
                     svg.setAttribute('width', '32');
@@ -109,13 +107,16 @@ export class CursorVisual {
                     document.body.appendChild(pointerElement);
                 }
                 //pointerElement.style.display = 'none';
+                pointerElement.style.transition = animate
+                    ? 'left 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), top 0.3s cubic-bezier(0.25, 0.1, 0.25, 1)'
+                    : 'none';
                 // Update position - use viewport coordinates for fixed positioning
                 // Set the top-left corner to (viewportX, viewportY) and then translate by (-1px, -3px)
                 // to align the pointer tip (approx. at 1.27, 4.17 within the SVG) with the click point.
                 pointerElement.style.left = `${viewportX}px`;
                 pointerElement.style.top = `${viewportY}px`;
                 pointerElement.style.transform = 'translate(-1px, -3px)';
-            }, { x, y, id: this.visualElementId, showClickEffect });
+            }, { x, y, id: this.visualElementId, showClickEffect, animate: this.animate });
         }
         catch (error) {
             // For example when:

package/dist/web/visualizer/cursor.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/web/visualizer/cursor.test.js

@@ -0,0 +1,63 @@
+import { expect, test } from 'bun:test';
+import { ActionVisualizer } from './index';
+import { CursorVisual } from './cursor';
+import { Operation } from '@/common/operation';
+import { OperationCancelledError } from '@/agent/errors';
+function pageFixture() {
+    const evaluations = [];
+    const page = {
+        on: () => { },
+        evaluate: async (_fn, arg) => { evaluations.push(arg); },
+    };
+    return { page, evaluations };
+}
+test('hidden cursor does not draw, register cursor timings, or wait', async () => {
+    const { page, evaluations } = pageFixture();
+    const visualizer = new ActionVisualizer({}, { showCursor: false });
+    await visualizer.setActivePage(page);
+    evaluations.length = 0;
+    const operation = new Operation({}, {});
+    await operation.run(async () => {
+        await visualizer.moveVirtualCursor(12, 34);
+        await visualizer.hideAll();
+        await visualizer.showAll();
+    });
+    operation.finish();
+    expect(evaluations).toEqual([]);
+    expect(operation.snapshot().timings.cursor).toBeUndefined();
+});
+test('animation can be disabled while preserving cursor position and visibility controls', async () => {
+    const { page, evaluations } = pageFixture();
+    const visualizer = new ActionVisualizer({}, { animateCursor: false });
+    await visualizer.setActivePage(page);
+    evaluations.length = 0;
+    const operation = new Operation({}, {});
+    // No timer is needed: the move finishes before the next timers phase.
+    const nextTimer = new Promise(resolve => setTimeout(() => resolve('timer'), 0));
+    const move = operation.run(() => visualizer.moveVirtualCursor(12, 34)).then(() => 'move');
+    expect(await Promise.race([move, nextTimer])).toBe('move');
+    await visualizer.hideAll();
+    await visualizer.showAll();
+    operation.finish();
+    expect(evaluations).toEqual([
+        { x: 12, y: 34, id: 'action-visual-indicator', showClickEffect: false, animate: false },
+        'action-visual-indicator', 'action-visual-indicator',
+    ]);
+    expect(operation.snapshot().timings.cursor?.count).toBe(1);
+});
+test('default animation still waits and its wait is cancellable', async () => {
+    const { page, evaluations } = pageFixture();
+    const cursor = new CursorVisual();
+    await cursor.setActivePage(page);
+    const operation = new Operation({}, {});
+    const move = operation.run(() => cursor.move(12, 34));
+    // Drawing is synchronous in this fixture; allow the animation wait to start.
+    await new Promise(resolve => setTimeout(resolve, 0));
+    expect(evaluations[0]).toMatchObject({ animate: true });
+    let complete = false;
+    void move.then(() => { complete = true; }, () => { });
+    expect(complete).toBe(false);
+    operation.cancel();
+    await expect(move).rejects.toBeInstanceOf(OperationCancelledError);
+    operation.finish();
+});

package/dist/web/visualizer/index.d.ts

@@ -1,6 +1,8 @@
 import { BrowserContext, Page } from "playwright";
 export interface ActionVisualizerOptions {
     showCursor?: boolean;
+    /** Set false to skip the 300 ms movement animation. Defaults to true. */
+    animateCursor?: boolean;
     showHoverCircle?: boolean;
     showClickRipple?: boolean;
     showDragLine?: boolean;

package/dist/web/visualizer/index.js

@@ -12,12 +12,13 @@ export class ActionVisualizer {
         this.context = context;
         this.options = {
             showCursor: options.showCursor ?? true,
+            animateCursor: options.animateCursor ?? true,
             showHoverCircle: options.showHoverCircle ?? false,
             showClickRipple: options.showClickRipple ?? true,
             showDragLine: options.showDragLine ?? false,
             showTypeEffects: options.showTypeEffects ?? true
         };
-        this.cursor = new CursorVisual();
+        this.cursor = new CursorVisual(this.options.animateCursor);
         this.mouseEffects = new MouseEffectVisual(this.options);
         this.typeEffects = new TypeEffectVisual();
     }
@@ -30,18 +31,21 @@ export class ActionVisualizer {
     }
     async setActivePage(page) {
         this.page = page;
-        await this.cursor.setActivePage(page);
+        if (this.options.showCursor)
+            await this.cursor.setActivePage(page);
         await this.mouseEffects.setActivePage(page);
         await this.typeEffects.setActivePage(page);
     }
     async moveVirtualCursor(x, y) {
-        // Takes like 300ms for smooth anim
-        await this.cursor.move(x, y);
+        if (this.options.showCursor)
+            await this.cursor.move(x, y);
     }
     async hideAll() {
-        await this.cursor.hide();
+        if (this.options.showCursor)
+            await this.cursor.hide();
     }
     async showAll() {
-        await this.cursor.show();
+        if (this.options.showCursor)
+            await this.cursor.show();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ddwang/magnitude-core",
-  "version": "0.3.1-ddwang.2",
+  "version": "0.3.1-ddwang.3",
   "description": "Magnitude e2e testing agent",
   "publishConfig": {
     "access": "public"