@midscene/visualizer 1.5.2 → 1.5.3-beta-20260305031559.0

package/dist/es/component/blackboard/index.css

@@ -21,21 +21,99 @@
   max-width: 500px;
 }
 
-.blackboard-filter {
-  margin: 10px 0;
+.blackboard-main-content {
+  position: relative;
+  overflow: hidden;
 }
 
-.blackboard-main-content canvas {
+.blackboard-screenshot {
   box-sizing: border-box;
   border: 1px solid #888;
   width: 100%;
+  display: block;
+}
+
+.blackboard-overlay {
+  pointer-events: none;
+  width: 100%;
+  position: absolute;
+  top: 0;
+  left: 0;
+}
+
+.blackboard-rect {
+  box-sizing: border-box;
+  pointer-events: none;
+  position: absolute;
+}
+
+.blackboard-rect-label {
+  white-space: nowrap;
+  padding: 1px 4px;
+  font-size: 14px;
+  font-weight: 600;
+  line-height: 1.4;
+  position: absolute;
+  bottom: 100%;
+  left: 0;
+}
+
+.blackboard-rect-search {
+  background: rgba(2, 131, 145, .4);
+  border: 1px solid #028391;
+  box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+}
+
+.blackboard-rect-search .blackboard-rect-label {
+  color: #028391;
+}
+
+.blackboard-rect-highlight {
+  background: rgba(253, 89, 7, .4);
+  border: 1px solid #fd5907;
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+  box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+}
+
+.blackboard-rect-highlight .blackboard-rect-label {
+  color: #000;
+}
+
+.blackboard-point {
+  pointer-events: none;
+  background: rgba(253, 89, 7, .4);
+  border: 1px solid #fd5907;
+  border-radius: 50%;
+  width: 20px;
+  height: 20px;
+  margin-top: -10px;
+  margin-left: -10px;
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+  position: absolute;
+  box-shadow: 0 0 8px rgba(253, 89, 7, .5);
+}
+
+@keyframes blackboard-pulse {
+  0%, 100% {
+    opacity: .4;
+    box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+  }
+
+  50% {
+    opacity: 1;
+    box-shadow: 4px 4px 2px rgba(51, 51, 51, .4), 0 0 16px rgba(253, 89, 7, .5);
+  }
 }
 
 [data-theme="dark"] .blackboard .footer, [data-theme="dark"] .blackboard .bottom-tip-item {
   color: rgba(255, 255, 255, .45);
 }
 
-[data-theme="dark"] .blackboard-main-content canvas {
+[data-theme="dark"] .blackboard-screenshot {
   border-color: rgba(255, 255, 255, .12);
 }
 
+[data-theme="dark"] .blackboard-rect-highlight .blackboard-rect-label {
+  color: #fff;
+}
+

package/dist/es/component/blackboard/index.mjs

@@ -1,91 +1,10 @@
 "use client";
 import { jsx, jsxs } from "react/jsx-runtime";
-import "pixi.js/unsafe-eval";
-import { Application, Container, Graphics, Rectangle, Sprite, Text, Texture } from "pixi.js";
-import { useEffect, useMemo, useRef, useState } from "react";
-import { colorForName, highlightColorForType } from "../../utils/color.mjs";
+import { useMemo, useRef } from "react";
 import "./index.css";
-import { DropShadowFilter, GlowFilter } from "pixi-filters";
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
-    try {
-        var info = gen[key](arg);
-        var value = info.value;
-    } catch (error) {
-        reject(error);
-        return;
-    }
-    if (info.done) resolve(value);
-    else Promise.resolve(value).then(_next, _throw);
-}
-function _async_to_generator(fn) {
-    return function() {
-        var self = this, args = arguments;
-        return new Promise(function(resolve, reject) {
-            var gen = fn.apply(self, args);
-            function _next(value) {
-                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
-            }
-            function _throw(err) {
-                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
-            }
-            _next(void 0);
-        });
-    };
-}
-const itemFillAlpha = 0.4;
-const highlightAlpha = 0.4;
-const pointRadius = 10;
-const pointMarkForItem = (point, type)=>{
-    const [x, y] = point;
-    const themeColor = highlightColorForType('element');
-    const graphics = new Graphics();
-    graphics.beginFill(themeColor, itemFillAlpha);
-    graphics.drawCircle(x, y, pointRadius);
-    graphics.endFill();
-    return graphics;
-};
-const rectMarkForItem = (rect, name, type)=>{
-    const { left, top, width, height } = rect;
-    let themeColor;
-    themeColor = 'element' === type ? colorForName(name) : 'searchArea' === type ? highlightColorForType('searchArea') : highlightColorForType('element');
-    const alpha = 'highlight' === type ? highlightAlpha : itemFillAlpha;
-    const graphics = new Graphics();
-    graphics.beginFill(themeColor, alpha);
-    graphics.lineStyle(1, themeColor, 1);
-    graphics.drawRect(left, top, width, height);
-    graphics.endFill();
-    const dropShadowFilter = new DropShadowFilter({
-        blur: 2,
-        quality: 3,
-        alpha: 0.4,
-        offset: {
-            x: 4,
-            y: 4
-        },
-        color: 0x333333
-    });
-    graphics.filters = [
-        dropShadowFilter
-    ];
-    const nameFontSize = 18;
-    if (!name) return [
-        graphics
-    ];
-    const texts = new Text(name, {
-        fontSize: nameFontSize,
-        fill: 0x0
-    });
-    texts.x = left;
-    texts.y = Math.max(top - (nameFontSize + 4), 0);
-    return [
-        graphics,
-        texts
-    ];
-};
 const Blackboard = (props)=>{
     var _props_uiContext;
     const highlightElements = props.highlightElements || [];
-    highlightElements.map((e)=>e.id);
     const highlightRect = props.highlightRect;
     const highlightPoints = props.highlightPoints;
     if (!(null == (_props_uiContext = props.uiContext) ? void 0 : _props_uiContext.shotSize)) return /*#__PURE__*/ jsx("div", {
@@ -100,6 +19,8 @@ const Blackboard = (props)=>{
     });
     const context = props.uiContext;
     const { shotSize, screenshot } = context;
+    const screenWidth = shotSize.width;
+    const screenHeight = shotSize.height;
     const screenshotBase64 = useMemo(()=>{
         if (!screenshot) return '';
         if ('object' == typeof screenshot && 'base64' in screenshot) return screenshot.base64;
@@ -108,221 +29,20 @@ const Blackboard = (props)=>{
     }, [
         screenshot
     ]);
-    const screenWidth = shotSize.width;
-    const screenHeight = shotSize.height;
-    const domRef = useRef(null);
-    const app = useMemo(()=>new Application(), []);
-    const [appInitialed, setAppInitialed] = useState(false);
-    const highlightContainer = useMemo(()=>new Container(), []);
-    const elementMarkContainer = useMemo(()=>new Container(), []);
-    const [hoverElement, setHoverElement] = useState(null);
-    const pixiBgRef = useRef(void 0);
-    const animationFrameRef = useRef(null);
-    const highlightGraphicsRef = useRef([]);
-    const glowFiltersRef = useRef([]);
-    const backgroundVisible = true;
-    const elementsVisible = true;
-    useEffect(()=>{
-        Promise.resolve((()=>_async_to_generator(function*() {
-                if (!domRef.current || !screenWidth) return;
-                yield app.init({
-                    width: screenWidth,
-                    height: screenHeight,
-                    background: 0xffffff
-                });
-                const canvasEl = domRef.current;
-                domRef.current.appendChild(app.canvas);
-                const { clientWidth } = domRef.current.parentElement;
-                const targetHeight = 0.6 * window.innerHeight;
-                const viewportRatio = clientWidth / targetHeight;
-                if (screenWidth / screenHeight <= viewportRatio) {
-                    const ratio = targetHeight / screenHeight;
-                    canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
-                    canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
-                }
-                app.stage.addChild(highlightContainer);
-                app.stage.addChild(elementMarkContainer);
-                setAppInitialed(true);
-            })())());
-        return ()=>{
-            console.log('will destroy');
-            if (null !== animationFrameRef.current) {
-                cancelAnimationFrame(animationFrameRef.current);
-                animationFrameRef.current = null;
-            }
-            try {
-                app.destroy(true, {
-                    children: true,
-                    texture: true
-                });
-            } catch (e) {
-                console.warn('destroy failed', e);
-            }
-        };
-    }, [
-        app,
-        screenWidth,
-        screenHeight
-    ]);
-    useEffect(()=>{
-        if (!appInitialed) return;
-        app.stage.eventMode = 'static';
-        app.stage.hitArea = new Rectangle(0, 0, screenWidth, screenHeight);
-        const clickHandler = (event)=>{
-            var _props_onCanvasClick;
-            console.log('pixi click', event);
-            const { x, y } = event.data.global;
-            null == (_props_onCanvasClick = props.onCanvasClick) || _props_onCanvasClick.call(props, [
-                Math.round(x),
-                Math.round(y)
-            ]);
-        };
-        app.stage.on('click', clickHandler);
-        return ()=>{
-            var _app_stage;
-            null == app || null == (_app_stage = app.stage) || _app_stage.off('click');
-        };
-    }, [
-        appInitialed,
-        props.onCanvasClick,
-        screenWidth,
-        screenHeight
-    ]);
-    useEffect(()=>{
-        if (!appInitialed) return;
-        const img = new Image();
-        img.onload = ()=>{
-            if (!app.stage) return;
-            const screenshotTexture = Texture.from(img);
-            const backgroundSprite = new Sprite(screenshotTexture);
-            backgroundSprite.x = 0;
-            backgroundSprite.y = 0;
-            backgroundSprite.width = screenWidth;
-            backgroundSprite.height = screenHeight;
-            backgroundSprite.eventMode = 'passive';
-            app.stage.addChildAt(backgroundSprite, 0);
-            pixiBgRef.current = backgroundSprite;
-            backgroundSprite.visible = backgroundVisible;
-        };
-        img.onerror = (e)=>{
-            console.error('load screenshot failed', e);
-        };
-        if (screenshotBase64) img.src = screenshotBase64;
-        else console.error('screenshotBase64 is empty, cannot load image');
-    }, [
-        app.stage,
-        appInitialed,
-        screenWidth,
-        screenHeight,
-        screenshotBase64
-    ]);
-    const { highlightElementRects } = useMemo(()=>{
-        const highlightElementRects = [];
-        highlightContainer.removeChildren();
-        elementMarkContainer.removeChildren();
-        highlightContainer.eventMode = 'passive';
-        elementMarkContainer.eventMode = 'passive';
-        highlightGraphicsRef.current = [];
-        glowFiltersRef.current = [];
-        if (highlightRect) {
-            const [graphics] = rectMarkForItem(highlightRect, 'Search Area', 'searchArea');
-            highlightContainer.addChild(graphics);
-        }
-        if (highlightElements.length) highlightElements.forEach((element)=>{
-            const { rect, content, id } = element;
-            const items = rectMarkForItem(rect, content, 'highlight');
-            const graphics = items[0];
-            const glowFilter = new GlowFilter({
-                distance: 30,
-                outerStrength: 3,
-                innerStrength: 0,
-                color: 0xfd5907,
-                quality: 0.5
-            });
-            const existingFilters = graphics.filters;
-            if (Array.isArray(existingFilters)) graphics.filters = [
-                ...existingFilters,
-                glowFilter
-            ];
-            else if (existingFilters) graphics.filters = [
-                existingFilters,
-                glowFilter
-            ];
-            else graphics.filters = [
-                glowFilter
-            ];
-            items.forEach((item)=>highlightContainer.addChild(item));
-            highlightGraphicsRef.current.push(graphics);
-            glowFiltersRef.current.push(glowFilter);
-        });
-        if (null == highlightPoints ? void 0 : highlightPoints.length) highlightPoints.forEach((point)=>{
-            const graphics = pointMarkForItem(point, 'highlightPoint');
-            const glowFilter = new GlowFilter({
-                distance: 25,
-                outerStrength: 2.5,
-                innerStrength: 0,
-                color: 0xfd5907,
-                quality: 0.5
-            });
-            graphics.filters = [
-                glowFilter
-            ];
-            highlightContainer.addChild(graphics);
-            highlightGraphicsRef.current.push(graphics);
-            glowFiltersRef.current.push(glowFilter);
-        });
-        elementMarkContainer.visible = elementsVisible;
-        return {
-            highlightElementRects
-        };
-    }, [
-        app,
-        appInitialed,
-        highlightElements,
-        hoverElement,
-        highlightRect,
-        highlightPoints
-    ]);
-    useEffect(()=>{
-        if (!appInitialed || 0 === highlightGraphicsRef.current.length) return void console.log('Animation skipped:', {
-            appInitialed,
-            graphicsCount: highlightGraphicsRef.current.length
-        });
-        console.log('Starting pulsing animation for', highlightGraphicsRef.current.length, 'graphics');
-        const graphicsToAnimate = highlightGraphicsRef.current;
-        const glowFilters = glowFiltersRef.current;
-        const pulseDuration = 1200;
-        const minAlpha = 0.4;
-        const maxAlpha = 1.0;
-        const minGlowStrength = 2.0;
-        const maxGlowStrength = 5.0;
-        const startTime = performance.now();
-        const animate = ()=>{
-            const elapsed = performance.now() - startTime;
-            const progress = elapsed % pulseDuration / pulseDuration;
-            const sineValue = Math.sin(progress * Math.PI * 2);
-            const normalizedSine = (sineValue + 1) / 2;
-            const alpha = minAlpha + normalizedSine * (maxAlpha - minAlpha);
-            const glowStrength = minGlowStrength + normalizedSine * (maxGlowStrength - minGlowStrength);
-            graphicsToAnimate.forEach((graphics, index)=>{
-                graphics.alpha = alpha;
-                if (glowFilters[index]) glowFilters[index].outerStrength = glowStrength;
-            });
-            animationFrameRef.current = requestAnimationFrame(animate);
-        };
-        animate();
-        return ()=>{
-            console.log('Stopping pulsing animation');
-            if (null !== animationFrameRef.current) {
-                cancelAnimationFrame(animationFrameRef.current);
-                animationFrameRef.current = null;
-            }
-        };
-    }, [
-        appInitialed,
-        highlightElements,
-        highlightPoints
-    ]);
+    const containerRef = useRef(null);
+    const handleClick = (e)=>{
+        if (!props.onCanvasClick || !containerRef.current) return;
+        const rect = containerRef.current.getBoundingClientRect();
+        const scaleX = screenWidth / rect.width;
+        const scaleY = screenHeight / rect.height;
+        const x = Math.round((e.clientX - rect.left) * scaleX);
+        const y = Math.round((e.clientY - rect.top) * scaleY);
+        props.onCanvasClick([
+            x,
+            y
+        ]);
+    };
+    const highlightElementRects = highlightElements.map((e)=>e.rect);
     let bottomTipA = null;
     if (1 === highlightElementRects.length) bottomTipA = /*#__PURE__*/ jsx("div", {
         className: "bottom-tip",
@@ -347,12 +67,64 @@ const Blackboard = (props)=>{
     return /*#__PURE__*/ jsxs("div", {
         className: "blackboard",
         children: [
-            /*#__PURE__*/ jsx("div", {
+            /*#__PURE__*/ jsxs("div", {
                 className: "blackboard-main-content",
+                ref: containerRef,
+                onClick: handleClick,
                 style: {
-                    width: '100%'
+                    width: '100%',
+                    position: 'relative',
+                    cursor: props.onCanvasClick ? 'crosshair' : void 0
                 },
-                ref: domRef
+                children: [
+                    screenshotBase64 && /*#__PURE__*/ jsx("img", {
+                        src: screenshotBase64,
+                        alt: "screenshot",
+                        className: "blackboard-screenshot",
+                        draggable: false
+                    }),
+                    /*#__PURE__*/ jsxs("div", {
+                        className: "blackboard-overlay",
+                        style: {
+                            aspectRatio: `${screenWidth}/${screenHeight}`
+                        },
+                        children: [
+                            highlightRect && /*#__PURE__*/ jsx("div", {
+                                className: "blackboard-rect blackboard-rect-search",
+                                style: {
+                                    left: `${highlightRect.left / screenWidth * 100}%`,
+                                    top: `${highlightRect.top / screenHeight * 100}%`,
+                                    width: `${highlightRect.width / screenWidth * 100}%`,
+                                    height: `${highlightRect.height / screenHeight * 100}%`
+                                },
+                                children: /*#__PURE__*/ jsx("span", {
+                                    className: "blackboard-rect-label",
+                                    children: "Search Area"
+                                })
+                            }),
+                            highlightElements.map((el, idx)=>/*#__PURE__*/ jsx("div", {
+                                    className: "blackboard-rect blackboard-rect-highlight",
+                                    style: {
+                                        left: `${el.rect.left / screenWidth * 100}%`,
+                                        top: `${el.rect.top / screenHeight * 100}%`,
+                                        width: `${el.rect.width / screenWidth * 100}%`,
+                                        height: `${el.rect.height / screenHeight * 100}%`
+                                    },
+                                    children: el.content && /*#__PURE__*/ jsx("span", {
+                                        className: "blackboard-rect-label",
+                                        children: el.content
+                                    })
+                                }, el.id || idx)),
+                            null == highlightPoints ? void 0 : highlightPoints.map((point, idx)=>/*#__PURE__*/ jsx("div", {
+                                    className: "blackboard-point",
+                                    style: {
+                                        left: `${point[0] / screenWidth * 100}%`,
+                                        top: `${point[1] / screenHeight * 100}%`
+                                    }
+                                }, idx))
+                        ]
+                    })
+                ]
             }),
             /*#__PURE__*/ jsx("div", {
                 className: "bottom-tip",
@@ -365,4 +137,4 @@ const Blackboard = (props)=>{
     });
 };
 const blackboard = Blackboard;
-export { Blackboard, blackboard as default, pointMarkForItem, rectMarkForItem };
+export { Blackboard, blackboard as default };