@midscene/visualizer 0.0.1 → 0.1.1

package/src/component/detail-panel.tsx

@@ -1,124 +0,0 @@
-'use client';
-import './detail-panel.less';
-import { Segmented, ConfigProvider } from 'antd';
-import { CameraOutlined, FileTextOutlined, ScheduleOutlined } from '@ant-design/icons';
-import { useEffect, useState } from 'react';
-import BlackBoard from './blackboard';
-import { useExecutionDump, useInsightDump } from '@/component/store';
-import { timeStr, filterBase64Value } from '@/utils';
-
-const ScreenshotItem = (props: { time: string; img: string }) => {
-  return (
-    <div className="screenshot-item">
-      <div className="screenshot-item-title">{props.time}</div>
-      <div>
-        <img src={props.img} />
-      </div>
-    </div>
-  );
-};
-
-const VIEW_TYPE_SCREENSHOT = 'screenshot';
-const VIEW_TYPE_JSON = 'json';
-const VIEW_TYPE_BLACKBOARD = 'blackboard';
-
-const DetailPanel = (): JSX.Element => {
-  const dumpId = useInsightDump((store) => store._loadId);
-  const blackboardViewAvailable = Boolean(dumpId);
-  const activeTask = useExecutionDump((store) => store.activeTask);
-  const [preferredViewType, setViewType] = useState(dumpId ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
-
-  const viewType =
-    preferredViewType === VIEW_TYPE_BLACKBOARD && !dumpId ? VIEW_TYPE_SCREENSHOT : preferredViewType;
-
-  let content;
-  if (!activeTask) {
-    content = <div>please select a task</div>;
-  } else if (viewType === VIEW_TYPE_JSON) {
-    content = (
-      <div className="json-content">{filterBase64Value(JSON.stringify(activeTask, undefined, 2))}</div>
-    );
-  } else if (viewType === VIEW_TYPE_BLACKBOARD) {
-    if (dumpId) {
-      content = <BlackBoard key={`${dumpId}`} />;
-    } else {
-      content = <div>invalid view</div>;
-    }
-  } else if (viewType === VIEW_TYPE_SCREENSHOT) {
-    if (activeTask.recorder?.length) {
-      content = (
-        <div>
-          {activeTask.recorder
-            .filter((item) => item.screenshot)
-            .map((item, index) => {
-              const fullTime = timeStr(item.ts);
-              const str = item.timing ? `${fullTime} / ${item.timing}` : fullTime;
-              return <ScreenshotItem key={index} time={str} img={item.screenshot!} />;
-            })}
-        </div>
-      );
-    } else {
-      content = <div>no screenshot</div>; // TODO: pretty error message
-    }
-  }
-
-  useEffect(() => {
-    // hit `Tab` to toggle viewType
-    const handleKeyDown = (e: KeyboardEvent) => {
-      if (e.key === 'Tab') {
-        if (viewType === VIEW_TYPE_BLACKBOARD) {
-          setViewType(VIEW_TYPE_SCREENSHOT);
-        } else if (viewType === VIEW_TYPE_SCREENSHOT) {
-          setViewType(VIEW_TYPE_JSON);
-        } else {
-          setViewType(blackboardViewAvailable ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
-        }
-        e.preventDefault();
-      }
-    };
-
-    document.addEventListener('keydown', handleKeyDown);
-
-    return () => {
-      document.removeEventListener('keydown', handleKeyDown);
-    };
-  });
-
-  const options = [
-    { label: 'Screenshots', value: VIEW_TYPE_SCREENSHOT, icon: <CameraOutlined /> },
-    { label: 'JSON View', value: VIEW_TYPE_JSON, icon: <FileTextOutlined /> },
-  ];
-  if (blackboardViewAvailable) {
-    options.unshift({ label: 'Visualization', value: VIEW_TYPE_BLACKBOARD, icon: <ScheduleOutlined /> });
-  }
-  return (
-    <div className="detail-panel">
-      <div className="view-switcher">
-        <ConfigProvider
-          theme={{
-            components: {
-              Segmented: {
-                itemSelectedBg: '#D7D7D7',
-                itemSelectedColor: '#000000',
-                // itemHoverColor: '#ffffff',
-                // itemHoverBg: '#A3D6D2', // @sub-blue
-                /* 这里是你的组件 token */
-              },
-            },
-          }}
-        >
-          <Segmented
-            options={options}
-            value={viewType}
-            onChange={(value: any) => {
-              setViewType(value);
-            }}
-          />
-        </ConfigProvider>
-      </div>
-      <div className="detail-content">{content}</div>
-    </div>
-  );
-};
-
-export default DetailPanel;

package/src/component/detail-side.less

@@ -1,131 +0,0 @@
-@import './common.less';
-
-@subSpacing: 5px;
-@lightTextColor: #777;
-
-.detail-side {
-  h2 {
-    padding-top: 0;
-    margin-top: 0;
-    margin-bottom: 4px;
-  }
-
-  .ant-tag{
-    margin-top: 2px;
-  }
-
-  .meta-kv {
-    padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
-    .meta {
-      box-sizing: border-box;
-      padding: 2px 0;
-      width: 100%;
-      display: flex;
-      flex-direction: row;
-      line-height: 1.5;
-
-      .meta-key {
-        // font-weight: bold;
-        width:100px;
-        text-align: right;
-        padding-right: 10px;
-        // color: #999;
-      }
-      
-      .meta-value {
-        flex: 1;
-      }
-    }
-  }
-
-  .item-list {
-    padding: @side-vertical-spacing @side-horizontal-padding;
-
-    cursor:default;
-    margin-bottom: 10px;
-
-    .item {
-      padding: 16px @side-horizontal-padding @side-horizontal-padding;
-      transition: .1s;
-      border: 1px solid #DDD;
-      border-radius: @subSpacing;
-      margin-bottom: @side-vertical-spacing;
-      position: relative;
-
-      &.item-lite {
-        border: none;
-        padding: 0;
-      }
-    }
-
-    .item-highlight {
-      color: #FFF;
-      .subtitle {
-        color: #CCC;
-      }
-    }
-    
-    .item-extra {
-      position: absolute;
-      right: @side-horizontal-padding;
-      top: @side-horizontal-padding;
-      color: @lightTextColor;
-    }
-
-    .title-right-padding {
-      padding-right: 15px;
-    }
-  
-    .title {
-      font-size: 18px;
-      font-weight: bold;
-      margin-bottom: @subSpacing;
-
-      .title-tag {
-        display: inline-block;
-        margin-left: 6px;
-        color: @lightTextColor;
-        font-size: 14px;
-        line-height: 18px;
-      }
-    }
-
-    .subtitle {
-      font-weight: normal;
-      font-size: 14px;
-      color: #777;
-    }
-
-    .description {
-      margin-top: @subSpacing;
-    }
-  
-    .description-content {
-      font-size: 14px;
-      margin-top: @side-horizontal-padding;
-      white-space: break-spaces;
-      margin: 0;
-    }
-
-    .element-button:hover {
-      // font-weight: bold;
-      color: #fff;
-      background: @main-orange;
-    }
-
-    .section-button:hover {
-      color: #fff;
-      background: #01204E;
-    }
-  }
-
-  .context {
-    pre {
-      text-wrap: balance;
-    }
-  }
-
-  .item-list-space-up {
-    margin-top: @side-vertical-spacing;
-  }
-}

package/src/component/detail-side.tsx

@@ -1,361 +0,0 @@
-/* eslint-disable max-lines */
-'use client';
-import './detail-side.less';
-import { RadiusSettingOutlined } from '@ant-design/icons';
-import { Tooltip, Tag, Timeline, TimelineItemProps } from 'antd';
-import {
-  BaseElement,
-  ExecutionTaskAction,
-  ExecutionTaskInsightLocate,
-  ExecutionTaskInsightQuery,
-  ExecutionTaskPlanning,
-  UISection,
-} from '@midscene/core';
-import { highlightColorForType } from './color';
-import { useExecutionDump, useInsightDump } from './store';
-import PanelTitle from './panel-title';
-import { timeCostStrElement } from './misc';
-import { timeStr, typeStr } from '@/utils';
-
-const noop = () => {};
-const Card = (props: {
-  liteMode?: boolean;
-  highlightWithColor?: string;
-  title?: string;
-  subtitle?: string;
-  characteristic?: string;
-  onMouseEnter?: () => void;
-  onMouseLeave?: () => void;
-  content: any;
-}) => {
-  const { highlightWithColor, title, subtitle, onMouseEnter, onMouseLeave, content, characteristic } = props;
-  const titleTag = props.characteristic ? (
-    <div className="item-extra">
-      <div className="title-tag">
-        <Tooltip placement="bottomRight" title={characteristic} mouseEnterDelay={0}>
-          <span>
-            <RadiusSettingOutlined />
-          </span>
-        </Tooltip>
-      </div>
-    </div>
-  ) : null;
-
-  const titleRightPaddingClass = props.characteristic ? 'title-right-padding' : '';
-  const modeClass = props.liteMode ? 'item-lite' : '';
-  const highlightStyle = highlightWithColor ? { backgroundColor: highlightWithColor } : {};
-  return (
-    <div
-      className={`item ${modeClass} ${highlightWithColor ? 'item-highlight' : ''}`}
-      style={{ ...highlightStyle }}
-      onMouseEnter={onMouseEnter}
-      onMouseLeave={onMouseLeave}
-    >
-      {/* {extraSection} */}
-
-      <div className={`title ${titleRightPaddingClass}`} style={{ display: title ? 'block' : 'none' }}>
-        {title}
-        {titleTag}
-      </div>
-      <div className={`subtitle ${titleRightPaddingClass}`} style={{ display: subtitle ? 'block' : 'none' }}>
-        {subtitle}
-      </div>
-      <div className="description" style={{ display: content ? 'block' : 'none' }}>
-        {content}
-      </div>
-    </div>
-  );
-};
-
-const MetaKV = (props: { data: { key: string; content: string | JSX.Element }[] }) => {
-  return (
-    <div className="meta-kv">
-      {props.data.map((item, index) => {
-        return (
-          <div className="meta" key={index}>
-            <div className="meta-key">{item.key}</div>
-            <div className="meta-value">{item.content}</div>
-          </div>
-        );
-      })}
-    </div>
-  );
-};
-
-const objectWithoutKeys = (obj: Record<string, unknown>, keys: string[]) =>
-  Object.keys(obj).reduce((acc, key) => {
-    if (!keys.includes(key)) {
-      (acc as any)[key] = obj[key];
-    }
-    return acc;
-  }, {});
-
-const DetailSide = (): JSX.Element => {
-  const task = useExecutionDump((store) => store.activeTask);
-  const dump = useInsightDump((store) => store.data);
-  const { matchedSection: sections, matchedElement: elements } = dump || {};
-  const highlightSectionNames = useInsightDump((store) => store.highlightSectionNames);
-  const highlightElements = useInsightDump((store) => store.highlightElements);
-  const setHighlightSectionNames = useInsightDump((store) => store.setHighlightSectionNames);
-  const setHighlightElements = useInsightDump((store) => store.setHighlightElements);
-
-  const setHighlightSectionName = function (name: string) {
-    setHighlightSectionNames([name]);
-  };
-  const setHighlightElement = function (element: BaseElement) {
-    setHighlightElements([element]);
-  };
-
-  const unhighlightSection = function () {
-    setHighlightSectionNames([]);
-  };
-
-  const unhighlightElement = function () {
-    setHighlightElements([]);
-  };
-
-  const kv = function (data: Record<string, unknown>) {
-    const isElementItem = (value: unknown): value is BaseElement =>
-      Boolean(value) &&
-      typeof value === 'object' &&
-      typeof (value as any).content !== 'undefined' &&
-      Boolean((value as any).center) &&
-      Boolean((value as any).rect);
-
-    const isSectionItem = (value: unknown): value is UISection =>
-      Boolean(value) &&
-      typeof (value as any)?.sectionCharacteristics !== 'undefined' &&
-      typeof (value as any)?.rect !== 'undefined';
-
-    const elementEl = (value: BaseElement) => (
-      <span
-        onMouseEnter={() => {
-          setHighlightElement(value);
-        }}
-        onMouseLeave={unhighlightElement}
-      >
-        <Tag bordered={false} color="orange" className="element-button">
-          Element
-        </Tag>
-      </span>
-    );
-
-    const sectionEl = (value: UISection) => (
-      <span
-        onMouseEnter={() => {
-          setHighlightSectionName(value.name);
-        }}
-        onMouseLeave={unhighlightSection}
-      >
-        <Tag bordered={false} color="blue" className="section-button">
-          Section
-        </Tag>
-      </span>
-    );
-
-    if (Array.isArray(data) || typeof data !== 'object') {
-      return <pre className="description-content">{JSON.stringify(data, undefined, 2)}</pre>;
-    }
-
-    return Object.keys(data).map((key) => {
-      const value = data[key];
-      let content;
-      if (typeof value === 'object' && isElementItem(value)) {
-        content = elementEl(value);
-      } else if (Array.isArray(value) && value.some((item) => isElementItem(item))) {
-        content = value.map((item, index) => <span key={index}>{elementEl(item)}</span>);
-      } else if (typeof value === 'object' && isSectionItem(value)) {
-        content = sectionEl(value);
-      } else if (Array.isArray(value) && value.some((item) => isSectionItem(item))) {
-        content = value.map((item, index) => <span key={index}>{sectionEl(item)}</span>);
-      } else {
-        content = typeof value === 'string' ? value : JSON.stringify(value, undefined, 2);
-      }
-
-      return (
-        <pre className="description-content" key={key}>
-          {key}:&nbsp;{content}
-        </pre>
-      );
-    });
-  };
-
-  const metaKVElement = MetaKV({
-    data: [
-      {
-        key: 'Status',
-        content: task?.status || '',
-      },
-      {
-        key: 'Start',
-        content: timeStr(task?.timing?.start),
-      },
-      {
-        key: 'End',
-        content: timeStr(task?.timing?.end),
-      },
-      {
-        key: 'Time Cost',
-        content: timeCostStrElement(task?.timing?.cost),
-      },
-    ],
-  });
-
-  let taskParam: JSX.Element | null = null;
-  if (task?.type === 'Planning') {
-    taskParam = MetaKV({
-      data: [
-        { key: 'type', content: (task && typeStr(task)) || '' },
-        { key: 'param', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
-      ],
-    });
-  } else if (task?.type === 'Insight') {
-    taskParam = MetaKV({
-      data: [
-        { key: 'type', content: (task && typeStr(task)) || '' },
-        {
-          key: 'param',
-          content: JSON.stringify(
-            (task as ExecutionTaskInsightLocate)?.param?.prompt ||
-              (task as ExecutionTaskInsightQuery)?.param?.dataDemand,
-          ),
-        },
-      ],
-    });
-  } else if (task?.type === 'Action') {
-    taskParam = MetaKV({
-      data: [
-        { key: 'type', content: (task && typeStr(task)) || '' },
-        {
-          key: 'value',
-          content: JSON.stringify((task as ExecutionTaskAction)?.param?.value, undefined, 2),
-        },
-      ],
-    });
-  }
-
-  const matchedSectionsEl = sections?.length
-    ? sections.map((section) => {
-        const { name } = section;
-        const ifHighlight = highlightSectionNames.includes(name);
-
-        const kvToShow = objectWithoutKeys(section as Record<string, any>, [
-          'name',
-          'description',
-          'texts',
-          'rect',
-          'sectionCharacteristics',
-        ]);
-        const sectionKV = Object.keys(kvToShow).length ? kv(kvToShow) : null;
-        const highlightColor = ifHighlight ? highlightColorForType('section') : undefined;
-
-        return (
-          <Card
-            title={section.name}
-            highlightWithColor={highlightColor}
-            subtitle={section.description}
-            characteristic={section.sectionCharacteristics}
-            onMouseEnter={setHighlightSectionName.bind(this, name)}
-            onMouseLeave={unhighlightSection}
-            content={sectionKV}
-            key={name}
-          />
-        );
-      })
-    : null;
-
-  const matchedElementsEl = elements?.length
-    ? elements.map((element, idx) => {
-        const ifHighlight = highlightElements.includes(element);
-        const highlightColor = ifHighlight ? highlightColorForType('element') : undefined;
-
-        const elementKV = kv(
-          objectWithoutKeys(element as any, [
-            'content',
-            'rect',
-            'center',
-            'left',
-            'top',
-            'right',
-            'bottom',
-            'locator',
-          ]),
-        );
-
-        return (
-          <Card
-            title={element.content}
-            highlightWithColor={highlightColor}
-            subtitle=""
-            onMouseEnter={setHighlightElement.bind(this, element)}
-            onMouseLeave={unhighlightElement}
-            content={elementKV}
-            key={idx}
-          />
-        );
-      })
-    : null;
-
-  // const [showQuery, setShowQuery] = useState(false);
-
-  const errorSection = dump?.error ? (
-    <Card
-      liteMode={true}
-      title="Error"
-      onMouseEnter={noop}
-      onMouseLeave={noop}
-      content={
-        <pre className="description-content" style={{ color: '#F00' }}>
-          {dump.error}
-        </pre>
-      }
-    />
-  ) : null;
-
-  const dataCard = dump?.data ? (
-    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
-  ) : null;
-
-  const plans = (task as ExecutionTaskPlanning)?.output?.plans;
-  let timelineData: TimelineItemProps[] = [];
-  if (plans) {
-    timelineData = timelineData.concat(
-      plans.map((item) => {
-        return {
-          color: '#06B1AB',
-          children: (
-            <>
-              <p>
-                <b>{typeStr(item as any)}</b>
-              </p>
-              <p>{item.thought}</p>
-              <p>{item.param ? JSON.stringify(item.param || {}, undefined, 2) : null}</p>
-            </>
-          ),
-        };
-      }),
-    );
-  }
-
-  return (
-    <div className="detail-side">
-      {/* Meta */}
-      <PanelTitle title="Task Meta" />
-      {metaKVElement}
-      {/* Param  */}
-      <PanelTitle title="Param" />
-      {taskParam}
-      {/* Response */}
-      <PanelTitle title="Output" />
-      <div className="item-list item-list-space-up">
-        {errorSection}
-        {dataCard}
-        {matchedSectionsEl}
-        {matchedElementsEl}
-        <Timeline items={timelineData} />
-      </div>
-    </div>
-  );
-};
-
-export default DetailSide;

package/src/component/global-hover-preview.less

@@ -1,23 +0,0 @@
-@import './common.less';
-
-@max-size: 400px;
-.global-hover-preview {
-  position: fixed;
-  display: block;
-  max-width: @max-size;
-  max-height: @max-size;
-  overflow: hidden;
-  z-index: 10;
-  text-align: center;
-  border: 1px solid @border-color;
-  box-sizing: border-box;
-  background: @side-bg;
-  box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
-  
-  img {
-    max-width: @max-size;
-    max-height: @max-size;
-    width: auto;
-    height: auto;
-  }
-}

package/src/component/global-hover-preview.tsx

@@ -1,50 +0,0 @@
-import { useRef, useState } from 'react';
-import { useExecutionDump } from './store';
-import './global-hover-preview.less';
-
-const size = 400; // @max-size
-const GlobalHoverPreview = () => {
-  const wrapperRef = useRef<HTMLDivElement>(null);
-  const hoverTask = useExecutionDump((store) => store.hoverTask);
-  const hoverPreviewConfig = useExecutionDump((store) => store.hoverPreviewConfig);
-  const [imageW, setImageW] = useState(size);
-  const [imageH, setImageH] = useState(size);
-
-  const images = hoverTask?.recorder
-    ?.filter((item) => {
-      return item.screenshot;
-    })
-    .map((item) => item.screenshot);
-
-  const { x, y } = hoverPreviewConfig || {};
-  let left = 0;
-  let top = 0;
-
-  const shouldShow = images?.length && typeof x !== 'undefined' && typeof y !== 'undefined';
-  if (shouldShow) {
-    const { clientWidth, clientHeight } = document.body;
-    const widthInPractice = imageW >= imageH ? size : size * (imageW / imageH);
-    const heightInPractice = imageW >= imageH ? size * (imageH / imageW) : size;
-    left = x + widthInPractice > clientWidth ? clientWidth - widthInPractice : x;
-    top = y + heightInPractice > clientHeight ? clientHeight - heightInPractice : y;
-  }
-  // if x + size exceed the screen width, use (screenWidth - size) instead
-
-  return shouldShow ? (
-    <div className="global-hover-preview" style={{ left, top }} ref={wrapperRef}>
-      {images?.length ? (
-        <img
-          src={images[0]}
-          onLoad={(img) => {
-            const imgElement = img.target as HTMLImageElement;
-            const width = imgElement.naturalWidth;
-            const height = imgElement.naturalHeight;
-            setImageW(width);
-            setImageH(height);
-          }}
-        />
-      ) : null}
-    </div>
-  ) : null;
-};
-export default GlobalHoverPreview;

package/src/component/misc.tsx

@@ -1,20 +0,0 @@
-export function timeCostStrElement(timeCost?: number) {
-  let str: string;
-  if (typeof timeCost !== 'number') {
-    str = '- ms';
-  } else if (timeCost > 1000) {
-    str = `${(timeCost / 1000).toFixed(2)}s`;
-  } else {
-    str = `${timeCost}ms`;
-  }
-  return (
-    <span
-      style={{
-        fontVariantNumeric: 'tabular-nums',
-        fontFeatureSettings: 'tnum',
-      }}
-    >
-      {str}
-    </span>
-  );
-}