@midscene/visualizer 0.0.1

package/src/component/detail-side.tsx

@@ -0,0 +1,361 @@
+/* eslint-disable max-lines */
+'use client';
+import './detail-side.less';
+import { RadiusSettingOutlined } from '@ant-design/icons';
+import { Tooltip, Tag, Timeline, TimelineItemProps } from 'antd';
+import {
+  BaseElement,
+  ExecutionTaskAction,
+  ExecutionTaskInsightLocate,
+  ExecutionTaskInsightQuery,
+  ExecutionTaskPlanning,
+  UISection,
+} from '@midscene/core';
+import { highlightColorForType } from './color';
+import { useExecutionDump, useInsightDump } from './store';
+import PanelTitle from './panel-title';
+import { timeCostStrElement } from './misc';
+import { timeStr, typeStr } from '@/utils';
+
+const noop = () => {};
+const Card = (props: {
+  liteMode?: boolean;
+  highlightWithColor?: string;
+  title?: string;
+  subtitle?: string;
+  characteristic?: string;
+  onMouseEnter?: () => void;
+  onMouseLeave?: () => void;
+  content: any;
+}) => {
+  const { highlightWithColor, title, subtitle, onMouseEnter, onMouseLeave, content, characteristic } = props;
+  const titleTag = props.characteristic ? (
+    <div className="item-extra">
+      <div className="title-tag">
+        <Tooltip placement="bottomRight" title={characteristic} mouseEnterDelay={0}>
+          <span>
+            <RadiusSettingOutlined />
+          </span>
+        </Tooltip>
+      </div>
+    </div>
+  ) : null;
+
+  const titleRightPaddingClass = props.characteristic ? 'title-right-padding' : '';
+  const modeClass = props.liteMode ? 'item-lite' : '';
+  const highlightStyle = highlightWithColor ? { backgroundColor: highlightWithColor } : {};
+  return (
+    <div
+      className={`item ${modeClass} ${highlightWithColor ? 'item-highlight' : ''}`}
+      style={{ ...highlightStyle }}
+      onMouseEnter={onMouseEnter}
+      onMouseLeave={onMouseLeave}
+    >
+      {/* {extraSection} */}
+
+      <div className={`title ${titleRightPaddingClass}`} style={{ display: title ? 'block' : 'none' }}>
+        {title}
+        {titleTag}
+      </div>
+      <div className={`subtitle ${titleRightPaddingClass}`} style={{ display: subtitle ? 'block' : 'none' }}>
+        {subtitle}
+      </div>
+      <div className="description" style={{ display: content ? 'block' : 'none' }}>
+        {content}
+      </div>
+    </div>
+  );
+};
+
+const MetaKV = (props: { data: { key: string; content: string | JSX.Element }[] }) => {
+  return (
+    <div className="meta-kv">
+      {props.data.map((item, index) => {
+        return (
+          <div className="meta" key={index}>
+            <div className="meta-key">{item.key}</div>
+            <div className="meta-value">{item.content}</div>
+          </div>
+        );
+      })}
+    </div>
+  );
+};
+
+const objectWithoutKeys = (obj: Record<string, unknown>, keys: string[]) =>
+  Object.keys(obj).reduce((acc, key) => {
+    if (!keys.includes(key)) {
+      (acc as any)[key] = obj[key];
+    }
+    return acc;
+  }, {});
+
+const DetailSide = (): JSX.Element => {
+  const task = useExecutionDump((store) => store.activeTask);
+  const dump = useInsightDump((store) => store.data);
+  const { matchedSection: sections, matchedElement: elements } = dump || {};
+  const highlightSectionNames = useInsightDump((store) => store.highlightSectionNames);
+  const highlightElements = useInsightDump((store) => store.highlightElements);
+  const setHighlightSectionNames = useInsightDump((store) => store.setHighlightSectionNames);
+  const setHighlightElements = useInsightDump((store) => store.setHighlightElements);
+
+  const setHighlightSectionName = function (name: string) {
+    setHighlightSectionNames([name]);
+  };
+  const setHighlightElement = function (element: BaseElement) {
+    setHighlightElements([element]);
+  };
+
+  const unhighlightSection = function () {
+    setHighlightSectionNames([]);
+  };
+
+  const unhighlightElement = function () {
+    setHighlightElements([]);
+  };
+
+  const kv = function (data: Record<string, unknown>) {
+    const isElementItem = (value: unknown): value is BaseElement =>
+      Boolean(value) &&
+      typeof value === 'object' &&
+      typeof (value as any).content !== 'undefined' &&
+      Boolean((value as any).center) &&
+      Boolean((value as any).rect);
+
+    const isSectionItem = (value: unknown): value is UISection =>
+      Boolean(value) &&
+      typeof (value as any)?.sectionCharacteristics !== 'undefined' &&
+      typeof (value as any)?.rect !== 'undefined';
+
+    const elementEl = (value: BaseElement) => (
+      <span
+        onMouseEnter={() => {
+          setHighlightElement(value);
+        }}
+        onMouseLeave={unhighlightElement}
+      >
+        <Tag bordered={false} color="orange" className="element-button">
+          Element
+        </Tag>
+      </span>
+    );
+
+    const sectionEl = (value: UISection) => (
+      <span
+        onMouseEnter={() => {
+          setHighlightSectionName(value.name);
+        }}
+        onMouseLeave={unhighlightSection}
+      >
+        <Tag bordered={false} color="blue" className="section-button">
+          Section
+        </Tag>
+      </span>
+    );
+
+    if (Array.isArray(data) || typeof data !== 'object') {
+      return <pre className="description-content">{JSON.stringify(data, undefined, 2)}</pre>;
+    }
+
+    return Object.keys(data).map((key) => {
+      const value = data[key];
+      let content;
+      if (typeof value === 'object' && isElementItem(value)) {
+        content = elementEl(value);
+      } else if (Array.isArray(value) && value.some((item) => isElementItem(item))) {
+        content = value.map((item, index) => <span key={index}>{elementEl(item)}</span>);
+      } else if (typeof value === 'object' && isSectionItem(value)) {
+        content = sectionEl(value);
+      } else if (Array.isArray(value) && value.some((item) => isSectionItem(item))) {
+        content = value.map((item, index) => <span key={index}>{sectionEl(item)}</span>);
+      } else {
+        content = typeof value === 'string' ? value : JSON.stringify(value, undefined, 2);
+      }
+
+      return (
+        <pre className="description-content" key={key}>
+          {key}:&nbsp;{content}
+        </pre>
+      );
+    });
+  };
+
+  const metaKVElement = MetaKV({
+    data: [
+      {
+        key: 'Status',
+        content: task?.status || '',
+      },
+      {
+        key: 'Start',
+        content: timeStr(task?.timing?.start),
+      },
+      {
+        key: 'End',
+        content: timeStr(task?.timing?.end),
+      },
+      {
+        key: 'Time Cost',
+        content: timeCostStrElement(task?.timing?.cost),
+      },
+    ],
+  });
+
+  let taskParam: JSX.Element | null = null;
+  if (task?.type === 'Planning') {
+    taskParam = MetaKV({
+      data: [
+        { key: 'type', content: (task && typeStr(task)) || '' },
+        { key: 'param', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
+      ],
+    });
+  } else if (task?.type === 'Insight') {
+    taskParam = MetaKV({
+      data: [
+        { key: 'type', content: (task && typeStr(task)) || '' },
+        {
+          key: 'param',
+          content: JSON.stringify(
+            (task as ExecutionTaskInsightLocate)?.param?.prompt ||
+              (task as ExecutionTaskInsightQuery)?.param?.dataDemand,
+          ),
+        },
+      ],
+    });
+  } else if (task?.type === 'Action') {
+    taskParam = MetaKV({
+      data: [
+        { key: 'type', content: (task && typeStr(task)) || '' },
+        {
+          key: 'value',
+          content: JSON.stringify((task as ExecutionTaskAction)?.param?.value, undefined, 2),
+        },
+      ],
+    });
+  }
+
+  const matchedSectionsEl = sections?.length
+    ? sections.map((section) => {
+        const { name } = section;
+        const ifHighlight = highlightSectionNames.includes(name);
+
+        const kvToShow = objectWithoutKeys(section as Record<string, any>, [
+          'name',
+          'description',
+          'texts',
+          'rect',
+          'sectionCharacteristics',
+        ]);
+        const sectionKV = Object.keys(kvToShow).length ? kv(kvToShow) : null;
+        const highlightColor = ifHighlight ? highlightColorForType('section') : undefined;
+
+        return (
+          <Card
+            title={section.name}
+            highlightWithColor={highlightColor}
+            subtitle={section.description}
+            characteristic={section.sectionCharacteristics}
+            onMouseEnter={setHighlightSectionName.bind(this, name)}
+            onMouseLeave={unhighlightSection}
+            content={sectionKV}
+            key={name}
+          />
+        );
+      })
+    : null;
+
+  const matchedElementsEl = elements?.length
+    ? elements.map((element, idx) => {
+        const ifHighlight = highlightElements.includes(element);
+        const highlightColor = ifHighlight ? highlightColorForType('element') : undefined;
+
+        const elementKV = kv(
+          objectWithoutKeys(element as any, [
+            'content',
+            'rect',
+            'center',
+            'left',
+            'top',
+            'right',
+            'bottom',
+            'locator',
+          ]),
+        );
+
+        return (
+          <Card
+            title={element.content}
+            highlightWithColor={highlightColor}
+            subtitle=""
+            onMouseEnter={setHighlightElement.bind(this, element)}
+            onMouseLeave={unhighlightElement}
+            content={elementKV}
+            key={idx}
+          />
+        );
+      })
+    : null;
+
+  // const [showQuery, setShowQuery] = useState(false);
+
+  const errorSection = dump?.error ? (
+    <Card
+      liteMode={true}
+      title="Error"
+      onMouseEnter={noop}
+      onMouseLeave={noop}
+      content={
+        <pre className="description-content" style={{ color: '#F00' }}>
+          {dump.error}
+        </pre>
+      }
+    />
+  ) : null;
+
+  const dataCard = dump?.data ? (
+    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
+  ) : null;
+
+  const plans = (task as ExecutionTaskPlanning)?.output?.plans;
+  let timelineData: TimelineItemProps[] = [];
+  if (plans) {
+    timelineData = timelineData.concat(
+      plans.map((item) => {
+        return {
+          color: '#06B1AB',
+          children: (
+            <>
+              <p>
+                <b>{typeStr(item as any)}</b>
+              </p>
+              <p>{item.thought}</p>
+              <p>{item.param ? JSON.stringify(item.param || {}, undefined, 2) : null}</p>
+            </>
+          ),
+        };
+      }),
+    );
+  }
+
+  return (
+    <div className="detail-side">
+      {/* Meta */}
+      <PanelTitle title="Task Meta" />
+      {metaKVElement}
+      {/* Param  */}
+      <PanelTitle title="Param" />
+      {taskParam}
+      {/* Response */}
+      <PanelTitle title="Output" />
+      <div className="item-list item-list-space-up">
+        {errorSection}
+        {dataCard}
+        {matchedSectionsEl}
+        {matchedElementsEl}
+        <Timeline items={timelineData} />
+      </div>
+    </div>
+  );
+};
+
+export default DetailSide;

package/src/component/global-hover-preview.less

@@ -0,0 +1,23 @@
+@import './common.less';
+
+@max-size: 400px;
+.global-hover-preview {
+  position: fixed;
+  display: block;
+  max-width: @max-size;
+  max-height: @max-size;
+  overflow: hidden;
+  z-index: 10;
+  text-align: center;
+  border: 1px solid @border-color;
+  box-sizing: border-box;
+  background: @side-bg;
+  box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
+  
+  img {
+    max-width: @max-size;
+    max-height: @max-size;
+    width: auto;
+    height: auto;
+  }
+}

package/src/component/global-hover-preview.tsx

@@ -0,0 +1,50 @@
+import { useRef, useState } from 'react';
+import { useExecutionDump } from './store';
+import './global-hover-preview.less';
+
+const size = 400; // @max-size
+const GlobalHoverPreview = () => {
+  const wrapperRef = useRef<HTMLDivElement>(null);
+  const hoverTask = useExecutionDump((store) => store.hoverTask);
+  const hoverPreviewConfig = useExecutionDump((store) => store.hoverPreviewConfig);
+  const [imageW, setImageW] = useState(size);
+  const [imageH, setImageH] = useState(size);
+
+  const images = hoverTask?.recorder
+    ?.filter((item) => {
+      return item.screenshot;
+    })
+    .map((item) => item.screenshot);
+
+  const { x, y } = hoverPreviewConfig || {};
+  let left = 0;
+  let top = 0;
+
+  const shouldShow = images?.length && typeof x !== 'undefined' && typeof y !== 'undefined';
+  if (shouldShow) {
+    const { clientWidth, clientHeight } = document.body;
+    const widthInPractice = imageW >= imageH ? size : size * (imageW / imageH);
+    const heightInPractice = imageW >= imageH ? size * (imageH / imageW) : size;
+    left = x + widthInPractice > clientWidth ? clientWidth - widthInPractice : x;
+    top = y + heightInPractice > clientHeight ? clientHeight - heightInPractice : y;
+  }
+  // if x + size exceed the screen width, use (screenWidth - size) instead
+
+  return shouldShow ? (
+    <div className="global-hover-preview" style={{ left, top }} ref={wrapperRef}>
+      {images?.length ? (
+        <img
+          src={images[0]}
+          onLoad={(img) => {
+            const imgElement = img.target as HTMLImageElement;
+            const width = imgElement.naturalWidth;
+            const height = imgElement.naturalHeight;
+            setImageW(width);
+            setImageH(height);
+          }}
+        />
+      ) : null}
+    </div>
+  ) : null;
+};
+export default GlobalHoverPreview;

package/src/component/misc.tsx

@@ -0,0 +1,20 @@
+export function timeCostStrElement(timeCost?: number) {
+  let str: string;
+  if (typeof timeCost !== 'number') {
+    str = '- ms';
+  } else if (timeCost > 1000) {
+    str = `${(timeCost / 1000).toFixed(2)}s`;
+  } else {
+    str = `${timeCost}ms`;
+  }
+  return (
+    <span
+      style={{
+        fontVariantNumeric: 'tabular-nums',
+        fontFeatureSettings: 'tnum',
+      }}
+    >
+      {str}
+    </span>
+  );
+}

package/src/component/panel-title.less

@@ -0,0 +1,11 @@
+@import './common.less';
+
+.task-list-name {
+  padding: 2px @side-horizontal-padding;
+  font-weight: bold;
+  background: @title-bg;
+  border-top: 1px solid @border-color;
+  border-bottom: 1px solid @border-color;
+  // box-shadow: 1px 1px 5px @border-color;
+  margin-top: -1px;
+}

package/src/component/panel-title.tsx

@@ -0,0 +1,11 @@
+import './panel-title.less';
+
+const PanelTitle = (props: { title: string }): JSX.Element => {
+  return (
+    <div>
+      <div className="task-list-name">{props.title}</div>
+    </div>
+  );
+};
+
+export default PanelTitle;

package/src/component/sidebar.less

@@ -0,0 +1,122 @@
+@import './common.less';
+
+.side-bar {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  width: 100%;
+  height: 100%;
+  border-right: 1px solid @border-color;
+  overflow: auto;
+  background: @side-bg;
+
+  .brand {
+    padding: @side-horizontal-padding 5px;
+    cursor: pointer;
+    // margin-bottom: 10px;
+    
+    // .head_name {
+    //   display: inline-block;
+    //   margin-left: 12px;
+    //   margin-right: 30px;
+    //   cursor: pointer;
+    //   font-weight: bold;
+    // }
+  }
+
+  .task-meta-section {
+    margin-top: 6px;
+  }
+
+  .task-meta {
+    color: @weak-text;
+    font-weight: normal;  
+    padding-left: @side-horizontal-padding;    
+  }
+
+  // side-seperator side-seperator-line side-seperator-space-up side-seperator-space-down" />
+  .side-seperator {
+    border-top: 1px solid none;
+    &.side-seperator-line {
+      border-top: 1px solid @border-color;
+    }
+
+    &.side-seperator-space-up {
+      margin-top: @side-vertical-spacing;
+    }
+
+    &.side-seperator-space-down {
+      margin-bottom: @side-vertical-spacing;
+    }
+  }
+
+  .side-sub-title {
+    padding: 0 @side-horizontal-padding;
+    font-weight: bold;
+    // margin-top: @side-vertical-spacing;
+    // color: @strong-text;
+  }
+
+  .side-item {
+    cursor: pointer;
+    transition: .1s;
+    // margin-bottom: 9px;
+    padding: 2px 0;
+    
+    &:hover {
+      background: @hover-bg;
+      color: @hover-text;
+    }
+    
+    &.selected
+    {
+      background: @selected-bg;
+      color: @selected-text;
+      
+      .status-text {
+        color: @selected-text;
+      }
+    }
+
+    .side-item-content {
+      padding: 0 @side-horizontal-padding 0 calc(@side-horizontal-padding + 16px);
+    }
+  }
+  
+  .side-item-name {
+    padding: 0 @side-horizontal-padding 0 calc(@side-horizontal-padding + 16px);
+    position: relative;
+    display: flex;
+    justify-content: space-between;
+    
+    .status-icon{
+      position: absolute;
+      left: @side-horizontal-padding;
+
+      display: inline-block;
+      color: #AAA;
+      font-size: 10px;
+      line-height: 10px;
+      top: 50%;
+      margin-top: -5px;
+    }
+    
+    .status-icon-success {
+      color: rgb(58, 119, 58);
+    }
+
+    .status-icon-fail {
+      color: rgb(255, 10, 10);
+    }
+
+    .status-text {
+      color: @weak-text;
+    }
+  }
+
+  .bottom-controls{
+    padding: @side-horizontal-padding 10px;
+    text-align: left;
+    text-align: center;
+  }
+}