@midscene/visualizer 0.0.1

package/dist/es/component/detail-side.css

@@ -0,0 +1,99 @@
+.detail-side h2 {
+  padding-top: 0;
+  margin-top: 0;
+  margin-bottom: 4px;
+}
+.detail-side .ant-tag {
+  margin-top: 2px;
+}
+.detail-side .meta-kv {
+  padding: 10px 10px calc(10px + 4px);
+}
+.detail-side .meta-kv .meta {
+  box-sizing: border-box;
+  padding: 2px 0;
+  width: 100%;
+  display: flex;
+  flex-direction: row;
+  line-height: 1.5;
+}
+.detail-side .meta-kv .meta .meta-key {
+  width: 100px;
+  text-align: right;
+  padding-right: 10px;
+}
+.detail-side .meta-kv .meta .meta-value {
+  flex: 1 1;
+}
+.detail-side .item-list {
+  padding: 10px 10px;
+  cursor: default;
+  margin-bottom: 10px;
+}
+.detail-side .item-list .item {
+  padding: 16px 10px 10px;
+  transition: 0.1s;
+  border: 1px solid #DDD;
+  border-radius: 5px;
+  margin-bottom: 10px;
+  position: relative;
+}
+.detail-side .item-list .item.item-lite {
+  border: none;
+  padding: 0;
+}
+.detail-side .item-list .item-highlight {
+  color: #FFF;
+}
+.detail-side .item-list .item-highlight .subtitle {
+  color: #CCC;
+}
+.detail-side .item-list .item-extra {
+  position: absolute;
+  right: 10px;
+  top: 10px;
+  color: #777;
+}
+.detail-side .item-list .title-right-padding {
+  padding-right: 15px;
+}
+.detail-side .item-list .title {
+  font-size: 18px;
+  font-weight: bold;
+  margin-bottom: 5px;
+}
+.detail-side .item-list .title .title-tag {
+  display: inline-block;
+  margin-left: 6px;
+  color: #777;
+  font-size: 14px;
+  line-height: 18px;
+}
+.detail-side .item-list .subtitle {
+  font-weight: normal;
+  font-size: 14px;
+  color: #777;
+}
+.detail-side .item-list .description {
+  margin-top: 5px;
+}
+.detail-side .item-list .description-content {
+  font-size: 14px;
+  margin-top: 10px;
+  white-space: break-spaces;
+  margin: 0;
+}
+.detail-side .item-list .element-button:hover {
+  color: #fff;
+  background: #F9483E;
+}
+.detail-side .item-list .section-button:hover {
+  color: #fff;
+  background: #01204E;
+}
+.detail-side .context pre {
+  text-wrap: balance;
+}
+.detail-side .item-list-space-up {
+  margin-top: 10px;
+}

package/dist/es/component/detail-side.js

@@ -0,0 +1,285 @@
+"use client";
+var __defProp = Object.defineProperty;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+import { Fragment, jsx, jsxs } from "react/jsx-runtime";
+import "./detail-side.css";
+import { RadiusSettingOutlined } from "@ant-design/icons";
+import { Tooltip, Tag, Timeline } from "antd";
+import { highlightColorForType } from "./color";
+import { useExecutionDump, useInsightDump } from "./store";
+import PanelTitle from "./panel-title";
+import { timeCostStrElement } from "./misc";
+import { timeStr, typeStr } from "../utils";
+const noop = () => {
+};
+const Card = (props) => {
+  const { highlightWithColor, title, subtitle, onMouseEnter, onMouseLeave, content, characteristic } = props;
+  const titleTag = props.characteristic ? /* @__PURE__ */ jsx("div", { className: "item-extra", children: /* @__PURE__ */ jsx("div", { className: "title-tag", children: /* @__PURE__ */ jsx(Tooltip, { placement: "bottomRight", title: characteristic, mouseEnterDelay: 0, children: /* @__PURE__ */ jsx("span", { children: /* @__PURE__ */ jsx(RadiusSettingOutlined, {}) }) }) }) }) : null;
+  const titleRightPaddingClass = props.characteristic ? "title-right-padding" : "";
+  const modeClass = props.liteMode ? "item-lite" : "";
+  const highlightStyle = highlightWithColor ? { backgroundColor: highlightWithColor } : {};
+  return /* @__PURE__ */ jsxs(
+    "div",
+    {
+      className: `item ${modeClass} ${highlightWithColor ? "item-highlight" : ""}`,
+      style: __spreadValues({}, highlightStyle),
+      onMouseEnter,
+      onMouseLeave,
+      children: [
+        /* @__PURE__ */ jsxs("div", { className: `title ${titleRightPaddingClass}`, style: { display: title ? "block" : "none" }, children: [
+          title,
+          titleTag
+        ] }),
+        /* @__PURE__ */ jsx("div", { className: `subtitle ${titleRightPaddingClass}`, style: { display: subtitle ? "block" : "none" }, children: subtitle }),
+        /* @__PURE__ */ jsx("div", { className: "description", style: { display: content ? "block" : "none" }, children: content })
+      ]
+    }
+  );
+};
+const MetaKV = (props) => {
+  return /* @__PURE__ */ jsx("div", { className: "meta-kv", children: props.data.map((item, index) => {
+    return /* @__PURE__ */ jsxs("div", { className: "meta", children: [
+      /* @__PURE__ */ jsx("div", { className: "meta-key", children: item.key }),
+      /* @__PURE__ */ jsx("div", { className: "meta-value", children: item.content })
+    ] }, index);
+  }) });
+};
+const objectWithoutKeys = (obj, keys) => Object.keys(obj).reduce((acc, key) => {
+  if (!keys.includes(key)) {
+    acc[key] = obj[key];
+  }
+  return acc;
+}, {});
+const DetailSide = () => {
+  var _a, _b, _c, _d, _e, _f, _g, _h;
+  const task = useExecutionDump((store) => store.activeTask);
+  const dump = useInsightDump((store) => store.data);
+  const { matchedSection: sections, matchedElement: elements } = dump || {};
+  const highlightSectionNames = useInsightDump((store) => store.highlightSectionNames);
+  const highlightElements = useInsightDump((store) => store.highlightElements);
+  const setHighlightSectionNames = useInsightDump((store) => store.setHighlightSectionNames);
+  const setHighlightElements = useInsightDump((store) => store.setHighlightElements);
+  const setHighlightSectionName = function(name) {
+    setHighlightSectionNames([name]);
+  };
+  const setHighlightElement = function(element) {
+    setHighlightElements([element]);
+  };
+  const unhighlightSection = function() {
+    setHighlightSectionNames([]);
+  };
+  const unhighlightElement = function() {
+    setHighlightElements([]);
+  };
+  const kv = function(data) {
+    const isElementItem = (value) => Boolean(value) && typeof value === "object" && typeof value.content !== "undefined" && Boolean(value.center) && Boolean(value.rect);
+    const isSectionItem = (value) => Boolean(value) && typeof (value == null ? void 0 : value.sectionCharacteristics) !== "undefined" && typeof (value == null ? void 0 : value.rect) !== "undefined";
+    const elementEl = (value) => /* @__PURE__ */ jsx(
+      "span",
+      {
+        onMouseEnter: () => {
+          setHighlightElement(value);
+        },
+        onMouseLeave: unhighlightElement,
+        children: /* @__PURE__ */ jsx(Tag, { bordered: false, color: "orange", className: "element-button", children: "Element" })
+      }
+    );
+    const sectionEl = (value) => /* @__PURE__ */ jsx(
+      "span",
+      {
+        onMouseEnter: () => {
+          setHighlightSectionName(value.name);
+        },
+        onMouseLeave: unhighlightSection,
+        children: /* @__PURE__ */ jsx(Tag, { bordered: false, color: "blue", className: "section-button", children: "Section" })
+      }
+    );
+    if (Array.isArray(data) || typeof data !== "object") {
+      return /* @__PURE__ */ jsx("pre", { className: "description-content", children: JSON.stringify(data, void 0, 2) });
+    }
+    return Object.keys(data).map((key) => {
+      const value = data[key];
+      let content;
+      if (typeof value === "object" && isElementItem(value)) {
+        content = elementEl(value);
+      } else if (Array.isArray(value) && value.some((item) => isElementItem(item))) {
+        content = value.map((item, index) => /* @__PURE__ */ jsx("span", { children: elementEl(item) }, index));
+      } else if (typeof value === "object" && isSectionItem(value)) {
+        content = sectionEl(value);
+      } else if (Array.isArray(value) && value.some((item) => isSectionItem(item))) {
+        content = value.map((item, index) => /* @__PURE__ */ jsx("span", { children: sectionEl(item) }, index));
+      } else {
+        content = typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+      }
+      return /* @__PURE__ */ jsxs("pre", { className: "description-content", children: [
+        key,
+        ": ",
+        content
+      ] }, key);
+    });
+  };
+  const metaKVElement = MetaKV({
+    data: [
+      {
+        key: "Status",
+        content: (task == null ? void 0 : task.status) || ""
+      },
+      {
+        key: "Start",
+        content: timeStr((_a = task == null ? void 0 : task.timing) == null ? void 0 : _a.start)
+      },
+      {
+        key: "End",
+        content: timeStr((_b = task == null ? void 0 : task.timing) == null ? void 0 : _b.end)
+      },
+      {
+        key: "Time Cost",
+        content: timeCostStrElement((_c = task == null ? void 0 : task.timing) == null ? void 0 : _c.cost)
+      }
+    ]
+  });
+  let taskParam = null;
+  if ((task == null ? void 0 : task.type) === "Planning") {
+    taskParam = MetaKV({
+      data: [
+        { key: "type", content: task && typeStr(task) || "" },
+        { key: "param", content: (_d = task == null ? void 0 : task.param) == null ? void 0 : _d.userPrompt }
+      ]
+    });
+  } else if ((task == null ? void 0 : task.type) === "Insight") {
+    taskParam = MetaKV({
+      data: [
+        { key: "type", content: task && typeStr(task) || "" },
+        {
+          key: "param",
+          content: JSON.stringify(
+            ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.prompt) || ((_f = task == null ? void 0 : task.param) == null ? void 0 : _f.dataDemand)
+          )
+        }
+      ]
+    });
+  } else if ((task == null ? void 0 : task.type) === "Action") {
+    taskParam = MetaKV({
+      data: [
+        { key: "type", content: task && typeStr(task) || "" },
+        {
+          key: "value",
+          content: JSON.stringify((_g = task == null ? void 0 : task.param) == null ? void 0 : _g.value, void 0, 2)
+        }
+      ]
+    });
+  }
+  const matchedSectionsEl = (sections == null ? void 0 : sections.length) ? sections.map((section) => {
+    const { name } = section;
+    const ifHighlight = highlightSectionNames.includes(name);
+    const kvToShow = objectWithoutKeys(section, [
+      "name",
+      "description",
+      "texts",
+      "rect",
+      "sectionCharacteristics"
+    ]);
+    const sectionKV = Object.keys(kvToShow).length ? kv(kvToShow) : null;
+    const highlightColor = ifHighlight ? highlightColorForType("section") : void 0;
+    return /* @__PURE__ */ jsx(
+      Card,
+      {
+        title: section.name,
+        highlightWithColor: highlightColor,
+        subtitle: section.description,
+        characteristic: section.sectionCharacteristics,
+        onMouseEnter: setHighlightSectionName.bind(void 0, name),
+        onMouseLeave: unhighlightSection,
+        content: sectionKV
+      },
+      name
+    );
+  }) : null;
+  const matchedElementsEl = (elements == null ? void 0 : elements.length) ? elements.map((element, idx) => {
+    const ifHighlight = highlightElements.includes(element);
+    const highlightColor = ifHighlight ? highlightColorForType("element") : void 0;
+    const elementKV = kv(
+      objectWithoutKeys(element, [
+        "content",
+        "rect",
+        "center",
+        "left",
+        "top",
+        "right",
+        "bottom",
+        "locator"
+      ])
+    );
+    return /* @__PURE__ */ jsx(
+      Card,
+      {
+        title: element.content,
+        highlightWithColor: highlightColor,
+        subtitle: "",
+        onMouseEnter: setHighlightElement.bind(void 0, element),
+        onMouseLeave: unhighlightElement,
+        content: elementKV
+      },
+      idx
+    );
+  }) : null;
+  const errorSection = (dump == null ? void 0 : dump.error) ? /* @__PURE__ */ jsx(
+    Card,
+    {
+      liteMode: true,
+      title: "Error",
+      onMouseEnter: noop,
+      onMouseLeave: noop,
+      content: /* @__PURE__ */ jsx("pre", { className: "description-content", style: { color: "#F00" }, children: dump.error })
+    }
+  ) : null;
+  const dataCard = (dump == null ? void 0 : dump.data) ? /* @__PURE__ */ jsx(Card, { liteMode: true, onMouseEnter: noop, onMouseLeave: noop, content: /* @__PURE__ */ jsx("pre", { children: kv(dump.data) }) }) : null;
+  const plans = (_h = task == null ? void 0 : task.output) == null ? void 0 : _h.plans;
+  let timelineData = [];
+  if (plans) {
+    timelineData = timelineData.concat(
+      plans.map((item) => {
+        return {
+          color: "#06B1AB",
+          children: /* @__PURE__ */ jsxs(Fragment, { children: [
+            /* @__PURE__ */ jsx("p", { children: /* @__PURE__ */ jsx("b", { children: typeStr(item) }) }),
+            /* @__PURE__ */ jsx("p", { children: item.thought }),
+            /* @__PURE__ */ jsx("p", { children: item.param ? JSON.stringify(item.param || {}, void 0, 2) : null })
+          ] })
+        };
+      })
+    );
+  }
+  return /* @__PURE__ */ jsxs("div", { className: "detail-side", children: [
+    /* @__PURE__ */ jsx(PanelTitle, { title: "Task Meta" }),
+    metaKVElement,
+    /* @__PURE__ */ jsx(PanelTitle, { title: "Param" }),
+    taskParam,
+    /* @__PURE__ */ jsx(PanelTitle, { title: "Output" }),
+    /* @__PURE__ */ jsxs("div", { className: "item-list item-list-space-up", children: [
+      errorSection,
+      dataCard,
+      matchedSectionsEl,
+      matchedElementsEl,
+      /* @__PURE__ */ jsx(Timeline, { items: timelineData })
+    ] })
+  ] });
+};
+var detail_side_default = DetailSide;
+export {
+  detail_side_default as default
+};

package/dist/es/component/global-hover-preview.css

@@ -0,0 +1,19 @@
+.global-hover-preview {
+  position: fixed;
+  display: block;
+  max-width: 400px;
+  max-height: 400px;
+  overflow: hidden;
+  z-index: 10;
+  text-align: center;
+  border: 1px solid #CCCCCC;
+  box-sizing: border-box;
+  background: #ECECEC;
+  box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
+}
+.global-hover-preview img {
+  max-width: 400px;
+  max-height: 400px;
+  width: auto;
+  height: auto;
+}

package/dist/es/component/global-hover-preview.js

@@ -0,0 +1,44 @@
+import { jsx } from "react/jsx-runtime";
+import { useRef, useState } from "react";
+import { useExecutionDump } from "./store";
+import "./global-hover-preview.css";
+const size = 400;
+const GlobalHoverPreview = () => {
+  var _a;
+  const wrapperRef = useRef(null);
+  const hoverTask = useExecutionDump((store) => store.hoverTask);
+  const hoverPreviewConfig = useExecutionDump((store) => store.hoverPreviewConfig);
+  const [imageW, setImageW] = useState(size);
+  const [imageH, setImageH] = useState(size);
+  const images = (_a = hoverTask == null ? void 0 : hoverTask.recorder) == null ? void 0 : _a.filter((item) => {
+    return item.screenshot;
+  }).map((item) => item.screenshot);
+  const { x, y } = hoverPreviewConfig || {};
+  let left = 0;
+  let top = 0;
+  const shouldShow = (images == null ? void 0 : images.length) && typeof x !== "undefined" && typeof y !== "undefined";
+  if (shouldShow) {
+    const { clientWidth, clientHeight } = document.body;
+    const widthInPractice = imageW >= imageH ? size : size * (imageW / imageH);
+    const heightInPractice = imageW >= imageH ? size * (imageH / imageW) : size;
+    left = x + widthInPractice > clientWidth ? clientWidth - widthInPractice : x;
+    top = y + heightInPractice > clientHeight ? clientHeight - heightInPractice : y;
+  }
+  return shouldShow ? /* @__PURE__ */ jsx("div", { className: "global-hover-preview", style: { left, top }, ref: wrapperRef, children: (images == null ? void 0 : images.length) ? /* @__PURE__ */ jsx(
+    "img",
+    {
+      src: images[0],
+      onLoad: (img) => {
+        const imgElement = img.target;
+        const width = imgElement.naturalWidth;
+        const height = imgElement.naturalHeight;
+        setImageW(width);
+        setImageH(height);
+      }
+    }
+  ) : null }) : null;
+};
+var global_hover_preview_default = GlobalHoverPreview;
+export {
+  global_hover_preview_default as default
+};

package/dist/es/component/misc.js

@@ -0,0 +1,24 @@
+import { jsx } from "react/jsx-runtime";
+function timeCostStrElement(timeCost) {
+  let str;
+  if (typeof timeCost !== "number") {
+    str = "- ms";
+  } else if (timeCost > 1e3) {
+    str = `${(timeCost / 1e3).toFixed(2)}s`;
+  } else {
+    str = `${timeCost}ms`;
+  }
+  return /* @__PURE__ */ jsx(
+    "span",
+    {
+      style: {
+        fontVariantNumeric: "tabular-nums",
+        fontFeatureSettings: "tnum"
+      },
+      children: str
+    }
+  );
+}
+export {
+  timeCostStrElement
+};

package/dist/es/component/panel-title.css

@@ -0,0 +1,8 @@
+.task-list-name {
+  padding: 2px 10px;
+  font-weight: bold;
+  background: #DDDDDD;
+  border-top: 1px solid #CCCCCC;
+  border-bottom: 1px solid #CCCCCC;
+  margin-top: -1px;
+}

package/dist/es/component/panel-title.js

@@ -0,0 +1,9 @@
+import { jsx } from "react/jsx-runtime";
+import "./panel-title.css";
+const PanelTitle = (props) => {
+  return /* @__PURE__ */ jsx("div", { children: /* @__PURE__ */ jsx("div", { className: "task-list-name", children: props.title }) });
+};
+var panel_title_default = PanelTitle;
+export {
+  panel_title_default as default
+};

package/dist/es/component/sidebar.css

@@ -0,0 +1,87 @@
+.side-bar {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  width: 100%;
+  height: 100%;
+  border-right: 1px solid #CCCCCC;
+  overflow: auto;
+  background: #ECECEC;
+}
+.side-bar .brand {
+  padding: 10px 5px;
+  cursor: pointer;
+}
+.side-bar .task-meta-section {
+  margin-top: 6px;
+}
+.side-bar .task-meta {
+  color: #777;
+  font-weight: normal;
+  padding-left: 10px;
+}
+.side-bar .side-seperator {
+  border-top: 1px solid none;
+}
+.side-bar .side-seperator.side-seperator-line {
+  border-top: 1px solid #CCCCCC;
+}
+.side-bar .side-seperator.side-seperator-space-up {
+  margin-top: 10px;
+}
+.side-bar .side-seperator.side-seperator-space-down {
+  margin-bottom: 10px;
+}
+.side-bar .side-sub-title {
+  padding: 0 10px;
+  font-weight: bold;
+}
+.side-bar .side-item {
+  cursor: pointer;
+  transition: 0.1s;
+  padding: 2px 0;
+}
+.side-bar .side-item:hover {
+  background: #A3D6D2;
+  color: #333;
+}
+.side-bar .side-item.selected {
+  background: #06B1AB;
+  color: #ffffff;
+}
+.side-bar .side-item.selected .status-text {
+  color: #ffffff;
+}
+.side-bar .side-item .side-item-content {
+  padding: 0 10px 0 calc(10px + 16px);
+}
+.side-bar .side-item-name {
+  padding: 0 10px 0 calc(10px + 16px);
+  position: relative;
+  display: flex;
+  justify-content: space-between;
+}
+.side-bar .side-item-name .status-icon {
+  position: absolute;
+  left: 10px;
+  display: inline-block;
+  color: #AAA;
+  font-size: 10px;
+  line-height: 10px;
+  top: 50%;
+  margin-top: -5px;
+}
+.side-bar .side-item-name .status-icon-success {
+  color: #3a773a;
+}
+.side-bar .side-item-name .status-icon-fail {
+  color: #ff0a0a;
+}
+.side-bar .side-item-name .status-text {
+  color: #777;
+}
+.side-bar .bottom-controls {
+  padding: 10px 10px;
+  text-align: left;
+  text-align: center;
+}