@midscene/visualizer 0.0.1

package/src/component/blackboard.less

@@ -0,0 +1,37 @@
+@import './common.less';
+
+.blackboard {
+  .footer {
+    color: #aaa;
+  }
+
+  ul {
+    padding-left: 0px;
+  }
+
+  li {
+    list-style: none;
+  }
+
+  .bottom-tip {
+    height: 30px;
+  }
+  
+  .bottom-tip-item {
+    max-width: 500px;
+    color: #AAA;
+    text-overflow: ellipsis;
+    word-wrap: break-word;
+  }
+}
+
+.blackboard-filter {
+  margin: 10px 0;
+}
+
+.blackboard-main-content {
+  canvas {
+    width: 100%;
+    border: 1px solid @heavy-border-color;
+  }
+}

package/src/component/blackboard.tsx

@@ -0,0 +1,293 @@
+'use client';
+import * as PIXI from 'pixi.js';
+import { ReactElement, useEffect, useMemo, useRef, useState } from 'react';
+import { Checkbox } from 'antd';
+import type { CheckboxProps } from 'antd';
+import { Rect } from '../../../midscene/dist/types';
+import { highlightColorForType, colorForName } from './color';
+import './blackboard.less';
+import { useBlackboardPreference, useInsightDump } from './store';
+
+const itemFillAlpha = 0.3;
+const bgOnAlpha = 0.8;
+const bgOffAlpha = 0.3;
+const noop = () => {
+  // noop
+};
+
+const BlackBoard = (): JSX.Element => {
+  const dump = useInsightDump((store) => store.data);
+  const setHighlightSectionNames = useInsightDump((store) => store.setHighlightSectionNames);
+  const setHighlightElements = useInsightDump((store) => store.setHighlightElements);
+  const highlightSectionNames = useInsightDump((store) => store.highlightSectionNames);
+  const highlightElements = useInsightDump((store) => store.highlightElements);
+
+  const { context, matchedSection: sections, matchedElement: elements } = dump!;
+  const { size, screenshotBase64 } = context;
+
+  const screenWidth = size.width;
+  const screenHeight = size.height;
+
+  const domRef = useRef<HTMLDivElement>(null); // Should be HTMLDivElement not HTMLInputElement
+  const app = useMemo<PIXI.Application>(() => new PIXI.Application(), []);
+  const [appInitialed, setAppInitialed] = useState(false);
+
+  const itemMarkContainer = useMemo(() => new PIXI.Container(), []);
+  const textContainer = useMemo(() => new PIXI.Container(), []);
+
+  // key overlays
+  const pixiBgRef = useRef<PIXI.Sprite>();
+  const { bgVisible, setBgVisible, textsVisible, setTextsVisible } = useBlackboardPreference();
+
+  useEffect(() => {
+    Promise.resolve(
+      (async () => {
+        if (!domRef.current || !screenWidth) {
+          return;
+        }
+        await app.init({ width: screenWidth, height: screenHeight, background: 0xffffff });
+        const canvasEl = domRef.current;
+        domRef.current.appendChild(app.canvas); // Ensure app.view is appended
+        const { clientWidth } = domRef.current.parentElement!;
+        const targetHeight = window.innerHeight * 0.7;
+        const viewportRatio = clientWidth / targetHeight;
+        if (screenWidth / screenHeight <= viewportRatio) {
+          const ratio = targetHeight / screenHeight;
+          canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
+          canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
+        }
+
+        app.stage.addChild(itemMarkContainer);
+        app.stage.addChild(textContainer);
+
+        setAppInitialed(true);
+      })(),
+    );
+
+    // Clean up the PIXI application when the component unmounts
+    return () => {
+      console.log('will destroy');
+      app?.destroy(true, { children: true, texture: true });
+    };
+  }, [app, screenWidth, screenHeight]);
+
+  // draw all texts on PIXI app
+  useEffect(() => {
+    if (!appInitialed) {
+      return;
+    }
+
+    // draw the screenshot base64
+    const img = new Image();
+    img.src = screenshotBase64;
+    img.onload = () => {
+      const screenshotTexture = PIXI.Texture.from(img);
+      const screenshotSprite = new PIXI.Sprite(screenshotTexture);
+      screenshotSprite.x = 0;
+      screenshotSprite.y = 0;
+      screenshotSprite.width = screenWidth;
+      screenshotSprite.height = screenHeight;
+      app.stage.addChildAt(screenshotSprite, 0);
+      pixiBgRef.current = screenshotSprite;
+      screenshotSprite.alpha = bgVisible ? bgOnAlpha : bgOffAlpha;
+    };
+  }, [app.stage, appInitialed]);
+
+  const rectMarkForItem = (
+    rect: Rect,
+    name: string,
+    themeColor: string,
+    alpha: number,
+    onPointOver: () => void,
+    onPointerOut: () => void,
+  ) => {
+    const { left, top, width, height } = rect;
+    const graphics = new PIXI.Graphics();
+    graphics.beginFill(themeColor, alpha);
+    graphics.lineStyle(1, themeColor, 1);
+    graphics.drawRect(left, top, width, height);
+    graphics.endFill();
+    graphics.interactive = true;
+    graphics.on('pointerover', onPointOver);
+    graphics.on('pointerout', onPointerOut);
+
+    const nameFontSize = 18;
+    const texts = new PIXI.Text(name, {
+      fontSize: nameFontSize,
+      fill: 0x0,
+    });
+    texts.x = left;
+    texts.y = Math.max(top - (nameFontSize + 4), 0);
+    return [graphics, texts];
+  };
+
+  const { highlightSectionRects, highlightElementRects } = useMemo(() => {
+    const highlightSectionRects: Rect[] = [];
+    const highlightElementRects: Rect[] = [];
+
+    itemMarkContainer.removeChildren();
+    textContainer.removeChildren();
+
+    sections.forEach((section) => {
+      // draw a section overlay
+      const ifHighlight = highlightSectionNames.includes(section.name);
+      if (ifHighlight) {
+        highlightSectionRects.push(section.rect);
+      }
+      const [graphics, texts] = rectMarkForItem(
+        section.rect,
+        section.name,
+        ifHighlight ? highlightColorForType('section') : colorForName('section', section.name),
+        ifHighlight ? 1 : itemFillAlpha,
+        () => {
+          setHighlightSectionNames([section.name]);
+        },
+        () => {
+          setHighlightSectionNames([]);
+        },
+      );
+      itemMarkContainer.addChild(graphics);
+      textContainer.addChild(texts);
+    });
+
+    // some are tmp highlights, draw them separately
+    highlightElements.forEach((element) => {
+      const { rect } = element;
+      highlightElementRects.push(rect);
+      if (elements.includes(element)) {
+        return;
+      }
+      const [graphics, texts] = rectMarkForItem(rect, '', highlightColorForType('element'), 1, noop, noop);
+      itemMarkContainer.addChild(graphics);
+      textContainer.addChild(texts);
+    });
+
+    // element mark
+    elements.forEach((element) => {
+      const { rect, content } = element;
+      const ifHighlight = highlightElements.includes(element);
+      const [graphics, texts] = rectMarkForItem(
+        rect,
+        content,
+        ifHighlight ? highlightColorForType('element') : colorForName('element', content),
+        ifHighlight ? 1 : itemFillAlpha,
+        () => {
+          setHighlightElements([element]);
+        },
+        () => {
+          setHighlightElements([]);
+        },
+      );
+      itemMarkContainer.addChild(graphics);
+      textContainer.addChild(texts);
+    });
+
+    sections.forEach((section) => {
+      const { content } = section;
+      const ifHighlight = highlightSectionNames.includes(section.name);
+
+      const sectionTheme = ifHighlight ? '#FFFFFF' : colorForName('section', section.name);
+
+      content.forEach((text) => {
+        const { content, rect } = text;
+        const { left, top } = rect;
+        const style = new PIXI.TextStyle({
+          wordWrap: true,
+          wordWrapWidth: rect.width,
+          fontSize: 18,
+          fill: sectionTheme,
+        });
+        const textElement = new PIXI.Text(content, style);
+        textElement.x = left;
+        textElement.y = top;
+        textContainer.addChild(textElement);
+
+        const textBorder = new PIXI.Graphics();
+        textBorder.beginFill(0xaaaaaa, 0.2);
+        textBorder.lineStyle(1, 0x0, 1);
+        textBorder.drawRect(left, top, rect.width, rect.height);
+        textBorder.endFill();
+        textContainer.addChild(textBorder);
+      });
+    });
+    textContainer.visible = textsVisible;
+    return {
+      highlightSectionRects,
+      highlightElementRects,
+    };
+  }, [
+    app,
+    appInitialed,
+    sections,
+    highlightSectionNames,
+    highlightElements,
+    // bgVisible,
+    // textsVisible,
+  ]);
+
+  const onSetBg: CheckboxProps['onChange'] = (e) => {
+    setBgVisible(e.target.checked);
+    if (pixiBgRef.current) {
+      pixiBgRef.current.alpha = e.target.checked ? bgOnAlpha : bgOffAlpha;
+    }
+  };
+
+  const onSetTextsVisible: CheckboxProps['onChange'] = (e) => {
+    setTextsVisible(e.target.checked);
+    textContainer.visible = e.target.checked;
+  };
+
+  let bottomTipA: ReactElement | null = null;
+  if (highlightElementRects.length === 1) {
+    bottomTipA = (
+      <div className="bottom-tip">
+        <div className="bottom-tip-item">Element: {JSON.stringify(highlightElementRects[0])}</div>
+      </div>
+    );
+  } else if (highlightElementRects.length > 1) {
+    bottomTipA = (
+      <div className="bottom-tip">
+        <div className="bottom-tip-item">Element: {JSON.stringify(highlightElementRects)}</div>
+      </div>
+    );
+  }
+
+  let bottomTipB: ReactElement | null = null;
+  if (highlightSectionRects.length === 1) {
+    bottomTipB = (
+      <div className="bottom-tip">
+        <div className="bottom-tip-item">Section: {JSON.stringify(highlightSectionRects[0])}</div>
+      </div>
+    );
+  } else if (highlightSectionRects.length > 1) {
+    bottomTipB = (
+      <div className="bottom-tip">
+        <div className="bottom-tip-item">Sections: {JSON.stringify(highlightSectionRects)}</div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="blackboard">
+      <div className="blackboard-main-content" style={{ width: '100%' }} ref={domRef} />
+      <div className="blackboard-filter">
+        <div className="overlay-control">
+          <Checkbox checked={bgVisible} onChange={onSetBg}>
+            Screenshot
+          </Checkbox>
+          <Checkbox checked={textsVisible} onChange={onSetTextsVisible}>
+            Text Mark
+          </Checkbox>
+        </div>
+      </div>
+      <div className="bottom-tip">
+        {bottomTipA}
+        {bottomTipB}
+      </div>
+
+      {/* {footer} */}
+    </div>
+  );
+};
+
+export default BlackBoard;

package/src/component/color.tsx

@@ -0,0 +1,34 @@
+// https://coolors.co/palettes/popular/#01204e
+const sectionColor = ['#028391'];
+const elementColor = ['#fb6107'];
+const highlightColorForSection = '#01204E';
+const highlightColorForElement = '#F56824'; // @main-orange
+
+function djb2Hash(str?: string): number {
+  if (!str) {
+    console.warn('djb2Hash: empty string');
+    str = 'unnamed';
+  }
+  let hash = 5381;
+  for (let i = 0; i < str.length; i++) {
+    hash = (hash << 5) + hash + str.charCodeAt(i); // hash * 33 + c
+  }
+  return hash >>> 0; // Convert to unsigned 32
+}
+
+export function colorForName(type: 'section' | 'element', name: string): string {
+  const hashNumber = djb2Hash(name);
+  if (type === 'section') {
+    return sectionColor[hashNumber % sectionColor.length];
+  } else {
+    return elementColor[hashNumber % elementColor.length];
+  }
+}
+
+export function highlightColorForType(type: 'section' | 'element'): string {
+  if (type === 'section') {
+    return highlightColorForSection;
+  } else {
+    return highlightColorForElement;
+  }
+}

package/src/component/common.less

@@ -0,0 +1,21 @@
+@main-blue: #06B1AB;
+@sub-blue: #A3D6D2;
+
+@main-orange: #F9483E;
+
+@side-bg: #ECECEC;
+@title-bg: #DDDDDD;
+@border-color: #CCCCCC;
+@heavy-border-color: #888;
+
+@selected-bg: @main-blue;
+@selected-text: #ffffff;
+@hover-bg: @sub-blue;
+@hover-text: #333;
+
+
+@weak-text: #777;
+@weak-bg: #F3F3F3;
+
+@side-horizontal-padding: 10px;
+@side-vertical-spacing: 10px;

package/src/component/detail-panel.less

@@ -0,0 +1,47 @@
+@import './common.less';
+
+@layout-space: 10px;
+
+.detail-panel {
+  display: flex;
+  flex-direction: column;
+  height: 100%;
+  padding: @layout-space;
+  background: #FFF;
+
+  .ant-segmented{
+    padding: 0;
+  }
+
+  .view-switcher{
+    margin-bottom: @layout-space;
+    flex-shrink: 0;
+  }
+
+  .detail-content{
+    box-sizing: border-box;
+    justify-content: center;
+    flex-direction: column;
+    display: flex;
+    flex-grow: 1;
+  }
+
+  .blackboard {
+    margin: 0 auto;
+  }
+
+  .screenshot-item {
+    margin-bottom: @layout-space;
+
+    .screenshot-item-title {
+      // font-weight: bold;
+      margin-bottom: 5px;
+    }
+
+    img {
+      border: 1px solid @heavy-border-color;
+      max-width: 100%;
+    }
+  }
+
+}

package/src/component/detail-panel.tsx

@@ -0,0 +1,124 @@
+'use client';
+import './detail-panel.less';
+import { Segmented, ConfigProvider } from 'antd';
+import { CameraOutlined, FileTextOutlined, ScheduleOutlined } from '@ant-design/icons';
+import { useEffect, useState } from 'react';
+import BlackBoard from './blackboard';
+import { useExecutionDump, useInsightDump } from '@/component/store';
+import { timeStr, filterBase64Value } from '@/utils';
+
+const ScreenshotItem = (props: { time: string; img: string }) => {
+  return (
+    <div className="screenshot-item">
+      <div className="screenshot-item-title">{props.time}</div>
+      <div>
+        <img src={props.img} />
+      </div>
+    </div>
+  );
+};
+
+const VIEW_TYPE_SCREENSHOT = 'screenshot';
+const VIEW_TYPE_JSON = 'json';
+const VIEW_TYPE_BLACKBOARD = 'blackboard';
+
+const DetailPanel = (): JSX.Element => {
+  const dumpId = useInsightDump((store) => store._loadId);
+  const blackboardViewAvailable = Boolean(dumpId);
+  const activeTask = useExecutionDump((store) => store.activeTask);
+  const [preferredViewType, setViewType] = useState(dumpId ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
+
+  const viewType =
+    preferredViewType === VIEW_TYPE_BLACKBOARD && !dumpId ? VIEW_TYPE_SCREENSHOT : preferredViewType;
+
+  let content;
+  if (!activeTask) {
+    content = <div>please select a task</div>;
+  } else if (viewType === VIEW_TYPE_JSON) {
+    content = (
+      <div className="json-content">{filterBase64Value(JSON.stringify(activeTask, undefined, 2))}</div>
+    );
+  } else if (viewType === VIEW_TYPE_BLACKBOARD) {
+    if (dumpId) {
+      content = <BlackBoard key={`${dumpId}`} />;
+    } else {
+      content = <div>invalid view</div>;
+    }
+  } else if (viewType === VIEW_TYPE_SCREENSHOT) {
+    if (activeTask.recorder?.length) {
+      content = (
+        <div>
+          {activeTask.recorder
+            .filter((item) => item.screenshot)
+            .map((item, index) => {
+              const fullTime = timeStr(item.ts);
+              const str = item.timing ? `${fullTime} / ${item.timing}` : fullTime;
+              return <ScreenshotItem key={index} time={str} img={item.screenshot!} />;
+            })}
+        </div>
+      );
+    } else {
+      content = <div>no screenshot</div>; // TODO: pretty error message
+    }
+  }
+
+  useEffect(() => {
+    // hit `Tab` to toggle viewType
+    const handleKeyDown = (e: KeyboardEvent) => {
+      if (e.key === 'Tab') {
+        if (viewType === VIEW_TYPE_BLACKBOARD) {
+          setViewType(VIEW_TYPE_SCREENSHOT);
+        } else if (viewType === VIEW_TYPE_SCREENSHOT) {
+          setViewType(VIEW_TYPE_JSON);
+        } else {
+          setViewType(blackboardViewAvailable ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
+        }
+        e.preventDefault();
+      }
+    };
+
+    document.addEventListener('keydown', handleKeyDown);
+
+    return () => {
+      document.removeEventListener('keydown', handleKeyDown);
+    };
+  });
+
+  const options = [
+    { label: 'Screenshots', value: VIEW_TYPE_SCREENSHOT, icon: <CameraOutlined /> },
+    { label: 'JSON View', value: VIEW_TYPE_JSON, icon: <FileTextOutlined /> },
+  ];
+  if (blackboardViewAvailable) {
+    options.unshift({ label: 'Visualization', value: VIEW_TYPE_BLACKBOARD, icon: <ScheduleOutlined /> });
+  }
+  return (
+    <div className="detail-panel">
+      <div className="view-switcher">
+        <ConfigProvider
+          theme={{
+            components: {
+              Segmented: {
+                itemSelectedBg: '#D7D7D7',
+                itemSelectedColor: '#000000',
+                // itemHoverColor: '#ffffff',
+                // itemHoverBg: '#A3D6D2', // @sub-blue
+                /* 这里是你的组件 token */
+              },
+            },
+          }}
+        >
+          <Segmented
+            options={options}
+            value={viewType}
+            onChange={(value: any) => {
+              setViewType(value);
+            }}
+          />
+        </ConfigProvider>
+      </div>
+      <div className="detail-content">{content}</div>
+    </div>
+  );
+};
+
+export default DetailPanel;

package/src/component/detail-side.less

@@ -0,0 +1,131 @@
+@import './common.less';
+
+@subSpacing: 5px;
+@lightTextColor: #777;
+
+.detail-side {
+  h2 {
+    padding-top: 0;
+    margin-top: 0;
+    margin-bottom: 4px;
+  }
+
+  .ant-tag{
+    margin-top: 2px;
+  }
+
+  .meta-kv {
+    padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
+    .meta {
+      box-sizing: border-box;
+      padding: 2px 0;
+      width: 100%;
+      display: flex;
+      flex-direction: row;
+      line-height: 1.5;
+
+      .meta-key {
+        // font-weight: bold;
+        width:100px;
+        text-align: right;
+        padding-right: 10px;
+        // color: #999;
+      }
+      
+      .meta-value {
+        flex: 1;
+      }
+    }
+  }
+
+  .item-list {
+    padding: @side-vertical-spacing @side-horizontal-padding;
+
+    cursor:default;
+    margin-bottom: 10px;
+
+    .item {
+      padding: 16px @side-horizontal-padding @side-horizontal-padding;
+      transition: .1s;
+      border: 1px solid #DDD;
+      border-radius: @subSpacing;
+      margin-bottom: @side-vertical-spacing;
+      position: relative;
+
+      &.item-lite {
+        border: none;
+        padding: 0;
+      }
+    }
+
+    .item-highlight {
+      color: #FFF;
+      .subtitle {
+        color: #CCC;
+      }
+    }
+    
+    .item-extra {
+      position: absolute;
+      right: @side-horizontal-padding;
+      top: @side-horizontal-padding;
+      color: @lightTextColor;
+    }
+
+    .title-right-padding {
+      padding-right: 15px;
+    }
+  
+    .title {
+      font-size: 18px;
+      font-weight: bold;
+      margin-bottom: @subSpacing;
+
+      .title-tag {
+        display: inline-block;
+        margin-left: 6px;
+        color: @lightTextColor;
+        font-size: 14px;
+        line-height: 18px;
+      }
+    }
+
+    .subtitle {
+      font-weight: normal;
+      font-size: 14px;
+      color: #777;
+    }
+
+    .description {
+      margin-top: @subSpacing;
+    }
+  
+    .description-content {
+      font-size: 14px;
+      margin-top: @side-horizontal-padding;
+      white-space: break-spaces;
+      margin: 0;
+    }
+
+    .element-button:hover {
+      // font-weight: bold;
+      color: #fff;
+      background: @main-orange;
+    }
+
+    .section-button:hover {
+      color: #fff;
+      background: #01204E;
+    }
+  }
+
+  .context {
+    pre {
+      text-wrap: balance;
+    }
+  }
+
+  .item-list-space-up {
+    margin-top: @side-vertical-spacing;
+  }
+}