@midscene/visualizer 0.26.2 → 0.26.3-beta-20250813021342.0

package/dist/es/blank_polyfill.mjs

@@ -0,0 +1,2 @@
+const AsyncLocalStorage = {};
+export { AsyncLocalStorage };

package/dist/es/component/blackboard.css

@@ -1,26 +1,33 @@
 .blackboard .footer {
   color: #aaa;
 }
+
 .blackboard ul {
-  padding-left: 0px;
+  padding-left: 0;
 }
+
 .blackboard li {
   list-style: none;
 }
+
 .blackboard .bottom-tip {
   height: 30px;
 }
+
 .blackboard .bottom-tip-item {
-  max-width: 500px;
-  color: #AAA;
+  color: #aaa;
   text-overflow: ellipsis;
   word-wrap: break-word;
+  max-width: 500px;
 }
+
 .blackboard-filter {
   margin: 10px 0;
 }
+
 .blackboard-main-content canvas {
-  width: 100%;
-  border: 1px solid #888;
   box-sizing: border-box;
+  border: 1px solid #888;
+  width: 100%;
 }
+

package/dist/es/component/blackboard.mjs

@@ -0,0 +1,266 @@
+'use client';
+import { jsx, jsxs } from "react/jsx-runtime";
+import "pixi.js/unsafe-eval";
+import { Checkbox } from "antd";
+import { Application, Container, Graphics, Rectangle, Sprite, Text, Texture } from "pixi.js";
+import { useEffect, useMemo, useRef, useState } from "react";
+import { colorForName, highlightColorForType } from "./color.mjs";
+import "./blackboard.css";
+import { treeToList } from "@midscene/shared/extractor";
+import { DropShadowFilter } from "pixi-filters";
+import { useBlackboardPreference } from "./store/store.mjs";
+const itemFillAlpha = 0.4;
+const highlightAlpha = 0.4;
+const pointRadius = 10;
+const pointMarkForItem = (point, type)=>{
+    const [x, y] = point;
+    const themeColor = highlightColorForType('element');
+    const graphics = new Graphics();
+    graphics.beginFill(themeColor, itemFillAlpha);
+    graphics.drawCircle(x, y, pointRadius);
+    graphics.endFill();
+    return graphics;
+};
+const rectMarkForItem = (rect, name, type)=>{
+    const { left, top, width, height } = rect;
+    let themeColor;
+    themeColor = 'element' === type ? colorForName(name) : 'searchArea' === type ? highlightColorForType('searchArea') : highlightColorForType('element');
+    const alpha = 'highlight' === type ? highlightAlpha : itemFillAlpha;
+    const graphics = new Graphics();
+    graphics.beginFill(themeColor, alpha);
+    graphics.lineStyle(1, themeColor, 1);
+    graphics.drawRect(left, top, width, height);
+    graphics.endFill();
+    const dropShadowFilter = new DropShadowFilter({
+        blur: 2,
+        quality: 3,
+        alpha: 0.4,
+        offset: {
+            x: 4,
+            y: 4
+        },
+        color: 0x333333
+    });
+    graphics.filters = [
+        dropShadowFilter
+    ];
+    const nameFontSize = 18;
+    if (!name) return [
+        graphics
+    ];
+    const texts = new Text(name, {
+        fontSize: nameFontSize,
+        fill: 0x0
+    });
+    texts.x = left;
+    texts.y = Math.max(top - (nameFontSize + 4), 0);
+    return [
+        graphics,
+        texts
+    ];
+};
+const Blackboard = (props)=>{
+    const highlightElements = props.highlightElements || [];
+    const highlightIds = highlightElements.map((e)=>e.id);
+    const highlightRect = props.highlightRect;
+    const highlightPoints = props.highlightPoints;
+    const context = props.uiContext;
+    const { size, screenshotBase64 } = context;
+    const screenWidth = size.width;
+    const screenHeight = size.height;
+    const domRef = useRef(null);
+    const app = useMemo(()=>new Application(), []);
+    const [appInitialed, setAppInitialed] = useState(false);
+    const highlightContainer = useMemo(()=>new Container(), []);
+    const elementMarkContainer = useMemo(()=>new Container(), []);
+    const [hoverElement, setHoverElement] = useState(null);
+    const pixiBgRef = useRef(void 0);
+    const { markerVisible, setMarkerVisible, elementsVisible, setTextsVisible } = useBlackboardPreference();
+    useEffect(()=>{
+        Promise.resolve((async ()=>{
+            if (!domRef.current || !screenWidth) return;
+            await app.init({
+                width: screenWidth,
+                height: screenHeight,
+                background: 0xffffff
+            });
+            const canvasEl = domRef.current;
+            domRef.current.appendChild(app.canvas);
+            const { clientWidth } = domRef.current.parentElement;
+            const targetHeight = 0.6 * window.innerHeight;
+            const viewportRatio = clientWidth / targetHeight;
+            if (screenWidth / screenHeight <= viewportRatio) {
+                const ratio = targetHeight / screenHeight;
+                canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
+                canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
+            }
+            app.stage.addChild(highlightContainer);
+            app.stage.addChild(elementMarkContainer);
+            setAppInitialed(true);
+        })());
+        return ()=>{
+            console.log('will destroy');
+            try {
+                app.destroy(true, {
+                    children: true,
+                    texture: true
+                });
+            } catch (e) {
+                console.warn('destroy failed', e);
+            }
+        };
+    }, [
+        app,
+        screenWidth,
+        screenHeight
+    ]);
+    useEffect(()=>{
+        if (!appInitialed) return;
+        app.stage.eventMode = 'static';
+        app.stage.hitArea = new Rectangle(0, 0, screenWidth, screenHeight);
+        const clickHandler = (event)=>{
+            var _props_onCanvasClick;
+            console.log('pixi click', event);
+            const { x, y } = event.data.global;
+            null == (_props_onCanvasClick = props.onCanvasClick) || _props_onCanvasClick.call(props, [
+                Math.round(x),
+                Math.round(y)
+            ]);
+        };
+        app.stage.on('click', clickHandler);
+        return ()=>{
+            var _app_stage;
+            null == app || null == (_app_stage = app.stage) || _app_stage.off('click');
+        };
+    }, [
+        appInitialed,
+        props.onCanvasClick,
+        screenWidth,
+        screenHeight
+    ]);
+    useEffect(()=>{
+        if (!appInitialed) return;
+        const img = new Image();
+        img.onload = ()=>{
+            if (!app.stage) return;
+            const screenshotTexture = Texture.from(img);
+            const backgroundSprite = new Sprite(screenshotTexture);
+            backgroundSprite.x = 0;
+            backgroundSprite.y = 0;
+            backgroundSprite.width = screenWidth;
+            backgroundSprite.height = screenHeight;
+            backgroundSprite.eventMode = 'passive';
+            app.stage.addChildAt(backgroundSprite, 0);
+            pixiBgRef.current = backgroundSprite;
+        };
+        img.onerror = (e)=>{
+            console.error('load screenshot failed', e);
+        };
+        img.src = screenshotBase64;
+    }, [
+        app.stage,
+        appInitialed,
+        screenWidth,
+        screenHeight
+    ]);
+    const { highlightElementRects } = useMemo(()=>{
+        const highlightElementRects = [];
+        highlightContainer.removeChildren();
+        elementMarkContainer.removeChildren();
+        highlightContainer.eventMode = 'passive';
+        elementMarkContainer.eventMode = 'passive';
+        if (highlightRect) {
+            const [graphics] = rectMarkForItem(highlightRect, 'Search Area', 'searchArea');
+            highlightContainer.addChild(graphics);
+        }
+        if (highlightElements.length) highlightElements.forEach((element)=>{
+            const { rect, content, id } = element;
+            const [graphics] = rectMarkForItem(rect, content, 'highlight');
+            highlightContainer.addChild(graphics);
+        });
+        if (null == highlightPoints ? void 0 : highlightPoints.length) highlightPoints.forEach((point)=>{
+            const graphics = pointMarkForItem(point, 'highlightPoint');
+            highlightContainer.addChild(graphics);
+        });
+        const elements = treeToList(context.tree);
+        elements.forEach((element)=>{
+            const { rect, content, id } = element;
+            const ifHighlight = highlightIds.includes(id) || (null == hoverElement ? void 0 : hoverElement.id) === id;
+            if (ifHighlight) return;
+            const [graphics] = rectMarkForItem(rect, content, 'element');
+            elementMarkContainer.addChild(graphics);
+        });
+        elementMarkContainer.visible = elementsVisible;
+        return {
+            highlightElementRects
+        };
+    }, [
+        app,
+        appInitialed,
+        highlightElements,
+        context.tree,
+        hoverElement,
+        highlightRect,
+        highlightPoints
+    ]);
+    const onSetElementsVisible = (e)=>{
+        setTextsVisible(e.target.checked);
+        elementMarkContainer.visible = e.target.checked;
+    };
+    let bottomTipA = null;
+    if (1 === highlightElementRects.length) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightElementRects[0])
+            ]
+        })
+    });
+    else if (highlightElementRects.length > 1) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightElementRects)
+            ]
+        })
+    });
+    return /*#__PURE__*/ jsxs("div", {
+        className: "blackboard",
+        children: [
+            /*#__PURE__*/ jsx("div", {
+                className: "blackboard-main-content",
+                style: {
+                    width: '100%'
+                },
+                ref: domRef
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "blackboard-filter",
+                style: {
+                    display: props.hideController ? 'none' : 'block'
+                },
+                children: /*#__PURE__*/ jsx("div", {
+                    className: "overlay-control",
+                    children: /*#__PURE__*/ jsx(Checkbox, {
+                        checked: elementsVisible,
+                        onChange: onSetElementsVisible,
+                        children: "Elements"
+                    })
+                })
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "bottom-tip",
+                style: {
+                    display: props.hideController ? 'none' : 'block'
+                },
+                children: bottomTipA
+            })
+        ]
+    });
+};
+const blackboard = Blackboard;
+export { Blackboard, blackboard as default, pointMarkForItem, rectMarkForItem };

package/dist/es/component/color.mjs

@@ -0,0 +1,35 @@
+const elementColor = [
+    '#01204E'
+];
+const highlightColorForSearchArea = '#028391';
+const highlightColorForElement = '#fd5907';
+function djb2Hash(str) {
+    if (!str) str = 'unnamed';
+    let hash = 5381;
+    for(let i = 0; i < str.length; i++)hash = (hash << 5) + hash + str.charCodeAt(i);
+    return hash >>> 0;
+}
+function colorForName(name) {
+    const hashNumber = djb2Hash(name);
+    return elementColor[hashNumber % elementColor.length];
+}
+function highlightColorForType(type) {
+    if ('searchArea' === type) return highlightColorForSearchArea;
+    return highlightColorForElement;
+}
+function globalThemeConfig() {
+    return {
+        token: {
+            colorPrimary: '#2B83FF'
+        },
+        components: {
+            Layout: {
+                headerHeight: 60,
+                headerPadding: '0 30px',
+                headerBg: '#FFF',
+                bodyBg: '#FFF'
+            }
+        }
+    };
+}
+export { colorForName, globalThemeConfig, highlightColorForType };

package/dist/es/component/describer.css

@@ -1,21 +1,25 @@
 .image-describer {
   position: relative;
 }
+
 .image-describer .describe-text {
   box-sizing: border-box;
-  position: absolute;
+  color: #fff;
   background: #000;
   width: 100%;
   height: 30px;
-  left: 0;
-  bottom: 0;
-  color: #FFF;
-  font-size: 12px;
   padding: 10px;
+  font-size: 12px;
+  position: absolute;
+  bottom: 0;
+  left: 0;
 }
+
 .image-describer .describe-text.success {
   background: #047704;
 }
+
 .image-describer .describe-text.error {
   background: #870707;
 }
+

package/dist/es/component/describer.mjs

@@ -0,0 +1,128 @@
+'use client';
+import { jsx, jsxs } from "react/jsx-runtime";
+import { useEffect, useRef, useState } from "react";
+import { useStaticPageAgent } from "./playground/useStaticPageAgent.mjs";
+import "./describer.css";
+import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
+import { Blackboard } from "./blackboard.mjs";
+import { PlaygroundResultView } from "./playground/PlaygroundResult.mjs";
+const Describer = (props)=>{
+    var _result_verifyResult;
+    const { uiContext } = props;
+    const image = uiContext.screenshotBase64;
+    const canvasRef = useRef(null);
+    const [highlightPoints, setHighlightPoints] = useState([]);
+    const [highlightRect, setHighlightRect] = useState();
+    const [error, setError] = useState();
+    const [loading, setLoading] = useState(false);
+    const [result, setResult] = useState();
+    const agent = useStaticPageAgent(uiContext);
+    useEffect(()=>{
+        const canvas = canvasRef.current;
+        if (!canvas || !image) return;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+        const img = new Image();
+        img.onload = ()=>{
+            canvas.width = img.width;
+            canvas.height = img.height;
+            ctx.drawImage(img, 0, 0);
+        };
+        img.src = image;
+    }, [
+        image
+    ]);
+    const handleClick = async (position)=>{
+        if (!agent) return void console.error('agent is not initialized');
+        setLoading(true);
+        setError(void 0);
+        setResult(void 0);
+        setHighlightPoints([]);
+        setHighlightRect(void 0);
+        try {
+            var _result_verifyResult;
+            const userLocation = [
+                position[0],
+                position[1]
+            ];
+            setHighlightPoints([
+                userLocation
+            ]);
+            const result = await (null == agent ? void 0 : agent.describeElementAtPoint(userLocation));
+            console.log('describe result', result);
+            setResult(result);
+            if (null == (_result_verifyResult = result.verifyResult) ? void 0 : _result_verifyResult.rect) setHighlightRect(result.verifyResult.rect);
+        } catch (error) {
+            setError(error.message);
+        } finally{
+            setLoading(false);
+        }
+    };
+    let resultText = '';
+    if (error) resultText = error;
+    else if (!result || (null == (_result_verifyResult = result.verifyResult) ? void 0 : _result_verifyResult.pass)) {
+        if (result) resultText = result.deepThink ? `Deep think: ${result.prompt}` : result.prompt;
+        else if (loading) resultText = 'Loading...';
+    } else resultText = `Locate failed with prompt: ${result.prompt}`;
+    return /*#__PURE__*/ jsx("div", {
+        className: "image-describer",
+        children: /*#__PURE__*/ jsxs(PanelGroup, {
+            autoSaveId: "describer-layout",
+            direction: "horizontal",
+            children: [
+                /*#__PURE__*/ jsx(Panel, {
+                    defaultSize: 32,
+                    maxSize: 60,
+                    minSize: 20,
+                    style: {
+                        paddingRight: '24px'
+                    },
+                    children: /*#__PURE__*/ jsxs("div", {
+                        className: "form-part context-panel",
+                        children: [
+                            /*#__PURE__*/ jsx("h3", {
+                                children: "Screenshot"
+                            }),
+                            /*#__PURE__*/ jsx("div", {
+                                className: "form-sub-title",
+                                children: "Click on the screenshot, Midscene will help you describe the element at the clicked point."
+                            }),
+                            /*#__PURE__*/ jsx(Blackboard, {
+                                uiContext: {
+                                    ...uiContext,
+                                    tree: {
+                                        node: null,
+                                        children: []
+                                    }
+                                },
+                                highlightPoints: highlightPoints,
+                                highlightRect: highlightRect,
+                                onCanvasClick: handleClick,
+                                hideController: true
+                            })
+                        ]
+                    })
+                }),
+                /*#__PURE__*/ jsx(PanelResizeHandle, {
+                    className: "panel-resize-handle"
+                }),
+                /*#__PURE__*/ jsx(Panel, {
+                    children: /*#__PURE__*/ jsx(PlaygroundResultView, {
+                        result: {
+                            result: resultText,
+                            error: error || null
+                        },
+                        loading: loading,
+                        serverValid: true,
+                        serviceMode: 'In-Browser',
+                        replayScriptsInfo: null,
+                        replayCounter: 0,
+                        loadingProgressText: ''
+                    })
+                })
+            ]
+        })
+    });
+};
+const describer = Describer;
+export { Describer, describer as default };

package/dist/es/component/env-config.mjs

@@ -0,0 +1,112 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { SettingOutlined } from "@ant-design/icons";
+import { Input, Modal, Tooltip } from "antd";
+import { useEffect, useRef, useState } from "react";
+import { useEnvConfig } from "./store/store.mjs";
+function EnvConfig(param) {
+    let { showTooltipWhenEmpty = true, showModelName = true, tooltipPlacement = 'bottom', mode = 'icon' } = param;
+    const { config, configString, loadConfig, syncFromStorage } = useEnvConfig();
+    const [isModalOpen, setIsModalOpen] = useState(false);
+    const [tempConfigString, setTempConfigString] = useState(configString);
+    const midsceneModelName = config.MIDSCENE_MODEL_NAME;
+    const componentRef = useRef(null);
+    const showModal = (e)=>{
+        syncFromStorage();
+        setIsModalOpen(true);
+        e.preventDefault();
+        e.stopPropagation();
+    };
+    const handleOk = ()=>{
+        setIsModalOpen(false);
+        loadConfig(tempConfigString);
+    };
+    const handleCancel = ()=>{
+        setIsModalOpen(false);
+    };
+    useEffect(()=>{
+        if (isModalOpen) setTempConfigString(configString);
+    }, [
+        isModalOpen,
+        configString
+    ]);
+    return /*#__PURE__*/ jsxs("div", {
+        style: {
+            display: 'flex',
+            justifyContent: 'flex-end',
+            gap: '10px',
+            alignItems: 'center',
+            height: '100%',
+            minHeight: '32px'
+        },
+        ref: componentRef,
+        children: [
+            showModelName ? midsceneModelName : null,
+            /*#__PURE__*/ jsx(Tooltip, {
+                title: "Please set up your environment variables before using.",
+                placement: tooltipPlacement,
+                align: {
+                    offset: [
+                        -10,
+                        5
+                    ]
+                },
+                getPopupContainer: ()=>componentRef.current,
+                open: isModalOpen ? false : showTooltipWhenEmpty ? 0 === Object.keys(config).length : void 0,
+                children: 'icon' === mode ? /*#__PURE__*/ jsx(SettingOutlined, {
+                    onClick: showModal
+                }) : /*#__PURE__*/ jsx("span", {
+                    onClick: showModal,
+                    style: {
+                        color: '#006AFF',
+                        cursor: 'pointer'
+                    },
+                    children: "set up"
+                })
+            }),
+            /*#__PURE__*/ jsxs(Modal, {
+                title: "Model Env Config",
+                open: isModalOpen,
+                onOk: handleOk,
+                onCancel: handleCancel,
+                okText: "Save",
+                style: {
+                    width: '800px',
+                    height: '100%',
+                    marginTop: '10%'
+                },
+                destroyOnClose: true,
+                maskClosable: true,
+                centered: true,
+                children: [
+                    /*#__PURE__*/ jsx(Input.TextArea, {
+                        rows: 7,
+                        placeholder: 'OPENAI_API_KEY=sk-...\nMIDSCENE_MODEL_NAME=gpt-4o-2024-08-06\n...',
+                        value: tempConfigString,
+                        onChange: (e)=>setTempConfigString(e.target.value),
+                        style: {
+                            whiteSpace: 'nowrap',
+                            wordWrap: 'break-word'
+                        }
+                    }),
+                    /*#__PURE__*/ jsxs("div", {
+                        children: [
+                            /*#__PURE__*/ jsx("p", {
+                                children: "The format is KEY=VALUE and separated by new lines."
+                            }),
+                            /*#__PURE__*/ jsxs("p", {
+                                children: [
+                                    "These data will be saved ",
+                                    /*#__PURE__*/ jsx("strong", {
+                                        children: "locally in your browser"
+                                    }),
+                                    "."
+                                ]
+                            })
+                        ]
+                    })
+                ]
+            })
+        ]
+    });
+}
+export { EnvConfig };

package/dist/es/component/github-star.css

@@ -1,3 +1,4 @@
 .github-star {
   height: 22px;
 }
+

package/dist/es/component/github-star.mjs

@@ -0,0 +1,20 @@
+import { jsx } from "react/jsx-runtime";
+import "./github-star.css";
+const GithubStar = ()=>/*#__PURE__*/ jsx("a", {
+        href: "https://github.com/web-infra-dev/midscene",
+        target: "_blank",
+        rel: "noreferrer",
+        style: {
+            display: 'flex',
+            alignItems: 'center'
+        },
+        children: /*#__PURE__*/ jsx("img", {
+            className: "github-star",
+            src: "https://img.shields.io/github/stars/web-infra-dev/midscene?style=social",
+            alt: "Github star",
+            style: {
+                display: 'block'
+            }
+        })
+    });
+export { GithubStar };

package/dist/es/component/logo.css

@@ -1,11 +1,13 @@
 .logo img {
-  height: 30px;
-  line-height: 30px;
   vertical-align: baseline;
+  height: 30px;
   vertical-align: -webkit-baseline-middle;
+  line-height: 30px;
 }
+
 .logo-with-star-wrapper {
-  display: flex;
   flex-direction: row;
   justify-content: space-between;
+  display: flex;
 }
+

package/dist/es/component/logo.mjs

@@ -0,0 +1,20 @@
+import { jsx } from "react/jsx-runtime";
+import "./logo.css";
+const LogoUrl = 'https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png';
+const Logo = (param)=>{
+    let { hideLogo = false } = param;
+    if (hideLogo) return null;
+    return /*#__PURE__*/ jsx("div", {
+        className: "logo",
+        children: /*#__PURE__*/ jsx("a", {
+            href: "https://midscenejs.com/",
+            target: "_blank",
+            rel: "noreferrer",
+            children: /*#__PURE__*/ jsx("img", {
+                alt: "Midscene_logo",
+                src: "https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png"
+            })
+        })
+    });
+};
+export { Logo, LogoUrl };