@midscene/visualizer 0.26.2-beta-20250812091127.0 → 0.26.3-beta-20250813021342.0

package/dist/es/component/blackboard.js

@@ -1,286 +0,0 @@
-"use client";
-var __async = (__this, __arguments, generator) => {
-  return new Promise((resolve, reject) => {
-    var fulfilled = (value) => {
-      try {
-        step(generator.next(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var rejected = (value) => {
-      try {
-        step(generator.throw(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
-    step((generator = generator.apply(__this, __arguments)).next());
-  });
-};
-import Buffer2 from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer: Buffer2
-};
-import { jsx, jsxs } from "react/jsx-runtime";
-import "pixi.js/unsafe-eval";
-import { Checkbox } from "antd";
-import * as PIXI from "pixi.js";
-import { useEffect, useMemo, useRef, useState } from "react";
-import { colorForName, highlightColorForType } from "./color";
-import "./blackboard.css";
-import { treeToList } from "@midscene/shared/extractor";
-import { DropShadowFilter } from "pixi-filters";
-import { useBlackboardPreference } from "./store/store";
-const itemFillAlpha = 0.4;
-const highlightAlpha = 0.4;
-const pointRadius = 10;
-const pointMarkForItem = (point, type) => {
-  const [x, y] = point;
-  const themeColor = highlightColorForType("element");
-  const graphics = new PIXI.Graphics();
-  graphics.beginFill(themeColor, itemFillAlpha);
-  graphics.drawCircle(x, y, pointRadius);
-  graphics.endFill();
-  return graphics;
-};
-const rectMarkForItem = (rect, name, type) => {
-  const { left, top, width, height } = rect;
-  let themeColor;
-  if (type === "element") {
-    themeColor = colorForName(name);
-  } else if (type === "searchArea") {
-    themeColor = highlightColorForType("searchArea");
-  } else {
-    themeColor = highlightColorForType("element");
-  }
-  const alpha = type === "highlight" ? highlightAlpha : itemFillAlpha;
-  const graphics = new PIXI.Graphics();
-  graphics.beginFill(themeColor, alpha);
-  graphics.lineStyle(1, themeColor, 1);
-  graphics.drawRect(left, top, width, height);
-  graphics.endFill();
-  const dropShadowFilter = new DropShadowFilter({
-    blur: 2,
-    quality: 3,
-    alpha: 0.4,
-    offset: { x: 4, y: 4 },
-    color: 3355443
-  });
-  graphics.filters = [dropShadowFilter];
-  const nameFontSize = 18;
-  if (!name) {
-    return [graphics];
-  }
-  const texts = new PIXI.Text(name, {
-    fontSize: nameFontSize,
-    fill: 0
-  });
-  texts.x = left;
-  texts.y = Math.max(top - (nameFontSize + 4), 0);
-  return [graphics, texts];
-};
-const Blackboard = (props) => {
-  const highlightElements = props.highlightElements || [];
-  const highlightIds = highlightElements.map((e) => e.id);
-  const highlightRect = props.highlightRect;
-  const highlightPoints = props.highlightPoints;
-  const context = props.uiContext;
-  const { size, screenshotBase64 } = context;
-  const screenWidth = size.width;
-  const screenHeight = size.height;
-  const domRef = useRef(null);
-  const app = useMemo(() => new PIXI.Application(), []);
-  const [appInitialed, setAppInitialed] = useState(false);
-  const highlightContainer = useMemo(() => new PIXI.Container(), []);
-  const elementMarkContainer = useMemo(() => new PIXI.Container(), []);
-  const [hoverElement, setHoverElement] = useState(null);
-  const pixiBgRef = useRef(void 0);
-  const { markerVisible, setMarkerVisible, elementsVisible, setTextsVisible } = useBlackboardPreference();
-  useEffect(() => {
-    Promise.resolve(
-      (() => __async(void 0, null, function* () {
-        if (!domRef.current || !screenWidth) {
-          return;
-        }
-        yield app.init({
-          width: screenWidth,
-          height: screenHeight,
-          background: 16777215
-        });
-        const canvasEl = domRef.current;
-        domRef.current.appendChild(app.canvas);
-        const { clientWidth } = domRef.current.parentElement;
-        const targetHeight = window.innerHeight * 0.6;
-        const viewportRatio = clientWidth / targetHeight;
-        if (screenWidth / screenHeight <= viewportRatio) {
-          const ratio = targetHeight / screenHeight;
-          canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
-          canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
-        }
-        app.stage.addChild(highlightContainer);
-        app.stage.addChild(elementMarkContainer);
-        setAppInitialed(true);
-      }))()
-    );
-    return () => {
-      console.log("will destroy");
-      try {
-        app.destroy(true, { children: true, texture: true });
-      } catch (e) {
-        console.warn("destroy failed", e);
-      }
-    };
-  }, [app, screenWidth, screenHeight]);
-  useEffect(() => {
-    if (!appInitialed) {
-      return;
-    }
-    app.stage.eventMode = "static";
-    app.stage.hitArea = new PIXI.Rectangle(0, 0, screenWidth, screenHeight);
-    const clickHandler = (event) => {
-      var _a;
-      console.log("pixi click", event);
-      const { x, y } = event.data.global;
-      (_a = props.onCanvasClick) == null ? void 0 : _a.call(props, [Math.round(x), Math.round(y)]);
-    };
-    app.stage.on("click", clickHandler);
-    return () => {
-      var _a;
-      (_a = app == null ? void 0 : app.stage) == null ? void 0 : _a.off("click");
-    };
-  }, [appInitialed, props.onCanvasClick, screenWidth, screenHeight]);
-  useEffect(() => {
-    if (!appInitialed) {
-      return;
-    }
-    const img = new Image();
-    img.onload = () => {
-      if (!app.stage)
-        return;
-      const screenshotTexture = PIXI.Texture.from(img);
-      const backgroundSprite = new PIXI.Sprite(screenshotTexture);
-      backgroundSprite.x = 0;
-      backgroundSprite.y = 0;
-      backgroundSprite.width = screenWidth;
-      backgroundSprite.height = screenHeight;
-      backgroundSprite.eventMode = "passive";
-      app.stage.addChildAt(backgroundSprite, 0);
-      pixiBgRef.current = backgroundSprite;
-    };
-    img.onerror = (e) => {
-      console.error("load screenshot failed", e);
-    };
-    img.src = screenshotBase64;
-  }, [app.stage, appInitialed, screenWidth, screenHeight]);
-  const { highlightElementRects } = useMemo(() => {
-    const highlightElementRects2 = [];
-    highlightContainer.removeChildren();
-    elementMarkContainer.removeChildren();
-    highlightContainer.eventMode = "passive";
-    elementMarkContainer.eventMode = "passive";
-    if (highlightRect) {
-      const [graphics] = rectMarkForItem(
-        highlightRect,
-        "Search Area",
-        "searchArea"
-      );
-      highlightContainer.addChild(graphics);
-    }
-    if (highlightElements.length) {
-      highlightElements.forEach((element) => {
-        const { rect, content, id } = element;
-        const [graphics] = rectMarkForItem(rect, content, "highlight");
-        highlightContainer.addChild(graphics);
-      });
-    }
-    if (highlightPoints == null ? void 0 : highlightPoints.length) {
-      highlightPoints.forEach((point) => {
-        const graphics = pointMarkForItem(point, "highlightPoint");
-        highlightContainer.addChild(graphics);
-      });
-    }
-    const elements = treeToList(context.tree);
-    elements.forEach((element) => {
-      const { rect, content, id } = element;
-      const ifHighlight = highlightIds.includes(id) || (hoverElement == null ? void 0 : hoverElement.id) === id;
-      if (ifHighlight) {
-        return;
-      }
-      const [graphics] = rectMarkForItem(rect, content, "element");
-      elementMarkContainer.addChild(graphics);
-    });
-    elementMarkContainer.visible = elementsVisible;
-    return {
-      highlightElementRects: highlightElementRects2
-    };
-  }, [
-    app,
-    appInitialed,
-    highlightElements,
-    context.tree,
-    hoverElement,
-    highlightRect,
-    highlightPoints
-    // bgVisible,
-    // elementsVisible,
-  ]);
-  const onSetMarkerVisible = (e) => {
-    setMarkerVisible(e.target.checked);
-    if (pixiBgRef.current) {
-      pixiBgRef.current.visible = e.target.checked;
-    }
-  };
-  const onSetElementsVisible = (e) => {
-    setTextsVisible(e.target.checked);
-    elementMarkContainer.visible = e.target.checked;
-  };
-  let bottomTipA = null;
-  if (highlightElementRects.length === 1) {
-    bottomTipA = /* @__PURE__ */ jsx("div", { className: "bottom-tip", children: /* @__PURE__ */ jsxs("div", { className: "bottom-tip-item", children: [
-      "Element: ",
-      JSON.stringify(highlightElementRects[0])
-    ] }) });
-  } else if (highlightElementRects.length > 1) {
-    bottomTipA = /* @__PURE__ */ jsx("div", { className: "bottom-tip", children: /* @__PURE__ */ jsxs("div", { className: "bottom-tip-item", children: [
-      "Element: ",
-      JSON.stringify(highlightElementRects)
-    ] }) });
-  }
-  return /* @__PURE__ */ jsxs("div", { className: "blackboard", children: [
-    /* @__PURE__ */ jsx(
-      "div",
-      {
-        className: "blackboard-main-content",
-        style: { width: "100%" },
-        ref: domRef
-      }
-    ),
-    /* @__PURE__ */ jsx(
-      "div",
-      {
-        className: "blackboard-filter",
-        style: { display: props.hideController ? "none" : "block" },
-        children: /* @__PURE__ */ jsx("div", { className: "overlay-control", children: /* @__PURE__ */ jsx(Checkbox, { checked: elementsVisible, onChange: onSetElementsVisible, children: "Elements" }) })
-      }
-    ),
-    /* @__PURE__ */ jsx(
-      "div",
-      {
-        className: "bottom-tip",
-        style: { display: props.hideController ? "none" : "block" },
-        children: bottomTipA
-      }
-    )
-  ] });
-};
-var blackboard_default = Blackboard;
-export {
-  Blackboard,
-  blackboard_default as default,
-  pointMarkForItem,
-  rectMarkForItem
-};

package/dist/es/component/color.js

@@ -1,49 +0,0 @@
-import Buffer from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer
-};
-const elementColor = ["#01204E"];
-const highlightColorForSearchArea = "#028391";
-const highlightColorForElement = "#fd5907";
-function djb2Hash(str) {
-  if (!str) {
-    str = "unnamed";
-  }
-  let hash = 5381;
-  for (let i = 0; i < str.length; i++) {
-    hash = (hash << 5) + hash + str.charCodeAt(i);
-  }
-  return hash >>> 0;
-}
-function colorForName(name) {
-  const hashNumber = djb2Hash(name);
-  return elementColor[hashNumber % elementColor.length];
-}
-function highlightColorForType(type) {
-  if (type === "searchArea") {
-    return highlightColorForSearchArea;
-  }
-  return highlightColorForElement;
-}
-function globalThemeConfig() {
-  return {
-    token: {
-      colorPrimary: "#2B83FF"
-    },
-    components: {
-      Layout: {
-        headerHeight: 60,
-        headerPadding: "0 30px",
-        headerBg: "#FFF",
-        bodyBg: "#FFF"
-      }
-    }
-  };
-}
-export {
-  colorForName,
-  globalThemeConfig,
-  highlightColorForType
-};

package/dist/es/component/describer.js

@@ -1,173 +0,0 @@
-"use client";
-var __defProp = Object.defineProperty;
-var __defProps = Object.defineProperties;
-var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
-var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __async = (__this, __arguments, generator) => {
-  return new Promise((resolve, reject) => {
-    var fulfilled = (value) => {
-      try {
-        step(generator.next(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var rejected = (value) => {
-      try {
-        step(generator.throw(value));
-      } catch (e) {
-        reject(e);
-      }
-    };
-    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
-    step((generator = generator.apply(__this, __arguments)).next());
-  });
-};
-import Buffer2 from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer: Buffer2
-};
-import { jsx, jsxs } from "react/jsx-runtime";
-import { useEffect, useRef, useState } from "react";
-import { useStaticPageAgent } from "./playground/useStaticPageAgent";
-import "./describer.css";
-import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
-import { Blackboard } from "./blackboard";
-import { PlaygroundResultView } from "./playground/PlaygroundResult";
-const Describer = (props) => {
-  var _a;
-  const { uiContext } = props;
-  const image = uiContext.screenshotBase64;
-  const canvasRef = useRef(null);
-  const [highlightPoints, setHighlightPoints] = useState(
-    []
-  );
-  const [highlightRect, setHighlightRect] = useState();
-  const [error, setError] = useState();
-  const [loading, setLoading] = useState(false);
-  const [result, setResult] = useState();
-  const agent = useStaticPageAgent(uiContext);
-  useEffect(() => {
-    const canvas = canvasRef.current;
-    if (!canvas || !image)
-      return;
-    const ctx = canvas.getContext("2d");
-    if (!ctx)
-      return;
-    const img = new Image();
-    img.onload = () => {
-      canvas.width = img.width;
-      canvas.height = img.height;
-      ctx.drawImage(img, 0, 0);
-    };
-    img.src = image;
-  }, [image]);
-  const handleClick = (position) => __async(void 0, null, function* () {
-    var _a2;
-    if (!agent) {
-      console.error("agent is not initialized");
-      return;
-    }
-    setLoading(true);
-    setError(void 0);
-    setResult(void 0);
-    setHighlightPoints([]);
-    setHighlightRect(void 0);
-    try {
-      const userLocation = [position[0], position[1]];
-      setHighlightPoints([userLocation]);
-      const result2 = yield agent == null ? void 0 : agent.describeElementAtPoint(userLocation);
-      console.log("describe result", result2);
-      setResult(result2);
-      if ((_a2 = result2.verifyResult) == null ? void 0 : _a2.rect) {
-        setHighlightRect(result2.verifyResult.rect);
-      }
-    } catch (error2) {
-      setError(error2.message);
-    } finally {
-      setLoading(false);
-    }
-  });
-  let resultText = "";
-  if (error) {
-    resultText = error;
-  } else if (result && !((_a = result.verifyResult) == null ? void 0 : _a.pass)) {
-    resultText = `Locate failed with prompt: ${result.prompt}`;
-  } else if (result) {
-    if (result.deepThink) {
-      resultText = `Deep think: ${result.prompt}`;
-    } else {
-      resultText = result.prompt;
-    }
-  } else if (loading) {
-    resultText = "Loading...";
-  }
-  return /* @__PURE__ */ jsx("div", { className: "image-describer", children: /* @__PURE__ */ jsxs(PanelGroup, { autoSaveId: "describer-layout", direction: "horizontal", children: [
-    /* @__PURE__ */ jsx(
-      Panel,
-      {
-        defaultSize: 32,
-        maxSize: 60,
-        minSize: 20,
-        style: { paddingRight: "24px" },
-        children: /* @__PURE__ */ jsxs("div", { className: "form-part context-panel", children: [
-          /* @__PURE__ */ jsx("h3", { children: "Screenshot" }),
-          /* @__PURE__ */ jsx("div", { className: "form-sub-title", children: "Click on the screenshot, Midscene will help you describe the element at the clicked point." }),
-          /* @__PURE__ */ jsx(
-            Blackboard,
-            {
-              uiContext: __spreadProps(__spreadValues({}, uiContext), {
-                tree: {
-                  node: null,
-                  children: []
-                }
-              }),
-              highlightPoints,
-              highlightRect,
-              onCanvasClick: handleClick,
-              hideController: true
-            }
-          )
-        ] })
-      }
-    ),
-    /* @__PURE__ */ jsx(PanelResizeHandle, { className: "panel-resize-handle" }),
-    /* @__PURE__ */ jsx(Panel, { children: /* @__PURE__ */ jsx(
-      PlaygroundResultView,
-      {
-        result: {
-          result: resultText,
-          error: error || null
-        },
-        loading,
-        serverValid: true,
-        serviceMode: "In-Browser",
-        replayScriptsInfo: null,
-        replayCounter: 0,
-        loadingProgressText: ""
-      }
-    ) })
-  ] }) });
-};
-var describer_default = Describer;
-export {
-  Describer,
-  describer_default as default
-};

package/dist/es/component/env-config.js

@@ -1,117 +0,0 @@
-import Buffer from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer
-};
-import { jsx, jsxs } from "react/jsx-runtime";
-import { SettingOutlined } from "@ant-design/icons";
-import { Input, Modal, Tooltip } from "antd";
-import { useEffect, useRef, useState } from "react";
-import { useEnvConfig } from "./store/store";
-function EnvConfig({
-  showTooltipWhenEmpty = true,
-  showModelName = true,
-  tooltipPlacement = "bottom",
-  mode = "icon"
-}) {
-  const { config, configString, loadConfig, syncFromStorage } = useEnvConfig();
-  const [isModalOpen, setIsModalOpen] = useState(false);
-  const [tempConfigString, setTempConfigString] = useState(configString);
-  const midsceneModelName = config.MIDSCENE_MODEL_NAME;
-  const componentRef = useRef(null);
-  const showModal = (e) => {
-    syncFromStorage();
-    setIsModalOpen(true);
-    e.preventDefault();
-    e.stopPropagation();
-  };
-  const handleOk = () => {
-    setIsModalOpen(false);
-    loadConfig(tempConfigString);
-  };
-  const handleCancel = () => {
-    setIsModalOpen(false);
-  };
-  useEffect(() => {
-    if (isModalOpen) {
-      setTempConfigString(configString);
-    }
-  }, [isModalOpen, configString]);
-  return /* @__PURE__ */ jsxs(
-    "div",
-    {
-      style: {
-        display: "flex",
-        justifyContent: "flex-end",
-        gap: "10px",
-        alignItems: "center",
-        height: "100%",
-        minHeight: "32px"
-      },
-      ref: componentRef,
-      children: [
-        showModelName ? midsceneModelName : null,
-        /* @__PURE__ */ jsx(
-          Tooltip,
-          {
-            title: "Please set up your environment variables before using.",
-            placement: tooltipPlacement,
-            align: { offset: [-10, 5] },
-            getPopupContainer: () => componentRef.current,
-            open: (
-              // undefined for default behavior of tooltip, hover for show
-              // close tooltip when modal is open
-              isModalOpen ? false : showTooltipWhenEmpty ? Object.keys(config).length === 0 : void 0
-            ),
-            children: mode === "icon" ? /* @__PURE__ */ jsx(SettingOutlined, { onClick: showModal }) : /* @__PURE__ */ jsx(
-              "span",
-              {
-                onClick: showModal,
-                style: { color: "#006AFF", cursor: "pointer" },
-                children: "set up"
-              }
-            )
-          }
-        ),
-        /* @__PURE__ */ jsxs(
-          Modal,
-          {
-            title: "Model Env Config",
-            open: isModalOpen,
-            onOk: handleOk,
-            onCancel: handleCancel,
-            okText: "Save",
-            style: { width: "800px", height: "100%", marginTop: "10%" },
-            destroyOnClose: true,
-            maskClosable: true,
-            centered: true,
-            children: [
-              /* @__PURE__ */ jsx(
-                Input.TextArea,
-                {
-                  rows: 7,
-                  placeholder: "OPENAI_API_KEY=sk-...\nMIDSCENE_MODEL_NAME=gpt-4o-2024-08-06\n...",
-                  value: tempConfigString,
-                  onChange: (e) => setTempConfigString(e.target.value),
-                  style: { whiteSpace: "nowrap", wordWrap: "break-word" }
-                }
-              ),
-              /* @__PURE__ */ jsxs("div", { children: [
-                /* @__PURE__ */ jsx("p", { children: "The format is KEY=VALUE and separated by new lines." }),
-                /* @__PURE__ */ jsxs("p", { children: [
-                  "These data will be saved ",
-                  /* @__PURE__ */ jsx("strong", { children: "locally in your browser" }),
-                  "."
-                ] })
-              ] })
-            ]
-          }
-        )
-      ]
-    }
-  );
-}
-export {
-  EnvConfig
-};

package/dist/es/component/github-star.js

@@ -1,31 +0,0 @@
-import Buffer from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer
-};
-import { jsx } from "react/jsx-runtime";
-import "./github-star.css";
-const GithubStar = () => {
-  return /* @__PURE__ */ jsx(
-    "a",
-    {
-      href: "https://github.com/web-infra-dev/midscene",
-      target: "_blank",
-      rel: "noreferrer",
-      style: { display: "flex", alignItems: "center" },
-      children: /* @__PURE__ */ jsx(
-        "img",
-        {
-          className: "github-star",
-          src: "https://img.shields.io/github/stars/web-infra-dev/midscene?style=social",
-          alt: "Github star",
-          style: { display: "block" }
-        }
-      )
-    }
-  );
-};
-export {
-  GithubStar
-};

package/dist/es/component/logo.js

@@ -1,25 +0,0 @@
-import Buffer from "buffer";
-import processBrowser from "process/browser";
-import console from "console-browserify";
-var buffer = {
-  Buffer
-};
-import { jsx } from "react/jsx-runtime";
-import "./logo.css";
-const LogoUrl = "https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png";
-const Logo = ({ hideLogo = false }) => {
-  if (hideLogo) {
-    return null;
-  }
-  return /* @__PURE__ */ jsx("div", { className: "logo", children: /* @__PURE__ */ jsx("a", { href: "https://midscenejs.com/", target: "_blank", rel: "noreferrer", children: /* @__PURE__ */ jsx(
-    "img",
-    {
-      alt: "Midscene_logo",
-      src: "https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png"
-    }
-  ) }) });
-};
-export {
-  Logo,
-  LogoUrl
-};