@midscene/visualizer 0.0.1 → 0.1.0

package/src/component/blackboard.tsx

@@ -1,293 +0,0 @@
-'use client';
-import * as PIXI from 'pixi.js';
-import { ReactElement, useEffect, useMemo, useRef, useState } from 'react';
-import { Checkbox } from 'antd';
-import type { CheckboxProps } from 'antd';
-import { Rect } from '../../../midscene/dist/types';
-import { highlightColorForType, colorForName } from './color';
-import './blackboard.less';
-import { useBlackboardPreference, useInsightDump } from './store';
-
-const itemFillAlpha = 0.3;
-const bgOnAlpha = 0.8;
-const bgOffAlpha = 0.3;
-const noop = () => {
-  // noop
-};
-
-const BlackBoard = (): JSX.Element => {
-  const dump = useInsightDump((store) => store.data);
-  const setHighlightSectionNames = useInsightDump((store) => store.setHighlightSectionNames);
-  const setHighlightElements = useInsightDump((store) => store.setHighlightElements);
-  const highlightSectionNames = useInsightDump((store) => store.highlightSectionNames);
-  const highlightElements = useInsightDump((store) => store.highlightElements);
-
-  const { context, matchedSection: sections, matchedElement: elements } = dump!;
-  const { size, screenshotBase64 } = context;
-
-  const screenWidth = size.width;
-  const screenHeight = size.height;
-
-  const domRef = useRef<HTMLDivElement>(null); // Should be HTMLDivElement not HTMLInputElement
-  const app = useMemo<PIXI.Application>(() => new PIXI.Application(), []);
-  const [appInitialed, setAppInitialed] = useState(false);
-
-  const itemMarkContainer = useMemo(() => new PIXI.Container(), []);
-  const textContainer = useMemo(() => new PIXI.Container(), []);
-
-  // key overlays
-  const pixiBgRef = useRef<PIXI.Sprite>();
-  const { bgVisible, setBgVisible, textsVisible, setTextsVisible } = useBlackboardPreference();
-
-  useEffect(() => {
-    Promise.resolve(
-      (async () => {
-        if (!domRef.current || !screenWidth) {
-          return;
-        }
-        await app.init({ width: screenWidth, height: screenHeight, background: 0xffffff });
-        const canvasEl = domRef.current;
-        domRef.current.appendChild(app.canvas); // Ensure app.view is appended
-        const { clientWidth } = domRef.current.parentElement!;
-        const targetHeight = window.innerHeight * 0.7;
-        const viewportRatio = clientWidth / targetHeight;
-        if (screenWidth / screenHeight <= viewportRatio) {
-          const ratio = targetHeight / screenHeight;
-          canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
-          canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
-        }
-
-        app.stage.addChild(itemMarkContainer);
-        app.stage.addChild(textContainer);
-
-        setAppInitialed(true);
-      })(),
-    );
-
-    // Clean up the PIXI application when the component unmounts
-    return () => {
-      console.log('will destroy');
-      app?.destroy(true, { children: true, texture: true });
-    };
-  }, [app, screenWidth, screenHeight]);
-
-  // draw all texts on PIXI app
-  useEffect(() => {
-    if (!appInitialed) {
-      return;
-    }
-
-    // draw the screenshot base64
-    const img = new Image();
-    img.src = screenshotBase64;
-    img.onload = () => {
-      const screenshotTexture = PIXI.Texture.from(img);
-      const screenshotSprite = new PIXI.Sprite(screenshotTexture);
-      screenshotSprite.x = 0;
-      screenshotSprite.y = 0;
-      screenshotSprite.width = screenWidth;
-      screenshotSprite.height = screenHeight;
-      app.stage.addChildAt(screenshotSprite, 0);
-      pixiBgRef.current = screenshotSprite;
-      screenshotSprite.alpha = bgVisible ? bgOnAlpha : bgOffAlpha;
-    };
-  }, [app.stage, appInitialed]);
-
-  const rectMarkForItem = (
-    rect: Rect,
-    name: string,
-    themeColor: string,
-    alpha: number,
-    onPointOver: () => void,
-    onPointerOut: () => void,
-  ) => {
-    const { left, top, width, height } = rect;
-    const graphics = new PIXI.Graphics();
-    graphics.beginFill(themeColor, alpha);
-    graphics.lineStyle(1, themeColor, 1);
-    graphics.drawRect(left, top, width, height);
-    graphics.endFill();
-    graphics.interactive = true;
-    graphics.on('pointerover', onPointOver);
-    graphics.on('pointerout', onPointerOut);
-
-    const nameFontSize = 18;
-    const texts = new PIXI.Text(name, {
-      fontSize: nameFontSize,
-      fill: 0x0,
-    });
-    texts.x = left;
-    texts.y = Math.max(top - (nameFontSize + 4), 0);
-    return [graphics, texts];
-  };
-
-  const { highlightSectionRects, highlightElementRects } = useMemo(() => {
-    const highlightSectionRects: Rect[] = [];
-    const highlightElementRects: Rect[] = [];
-
-    itemMarkContainer.removeChildren();
-    textContainer.removeChildren();
-
-    sections.forEach((section) => {
-      // draw a section overlay
-      const ifHighlight = highlightSectionNames.includes(section.name);
-      if (ifHighlight) {
-        highlightSectionRects.push(section.rect);
-      }
-      const [graphics, texts] = rectMarkForItem(
-        section.rect,
-        section.name,
-        ifHighlight ? highlightColorForType('section') : colorForName('section', section.name),
-        ifHighlight ? 1 : itemFillAlpha,
-        () => {
-          setHighlightSectionNames([section.name]);
-        },
-        () => {
-          setHighlightSectionNames([]);
-        },
-      );
-      itemMarkContainer.addChild(graphics);
-      textContainer.addChild(texts);
-    });
-
-    // some are tmp highlights, draw them separately
-    highlightElements.forEach((element) => {
-      const { rect } = element;
-      highlightElementRects.push(rect);
-      if (elements.includes(element)) {
-        return;
-      }
-      const [graphics, texts] = rectMarkForItem(rect, '', highlightColorForType('element'), 1, noop, noop);
-      itemMarkContainer.addChild(graphics);
-      textContainer.addChild(texts);
-    });
-
-    // element mark
-    elements.forEach((element) => {
-      const { rect, content } = element;
-      const ifHighlight = highlightElements.includes(element);
-      const [graphics, texts] = rectMarkForItem(
-        rect,
-        content,
-        ifHighlight ? highlightColorForType('element') : colorForName('element', content),
-        ifHighlight ? 1 : itemFillAlpha,
-        () => {
-          setHighlightElements([element]);
-        },
-        () => {
-          setHighlightElements([]);
-        },
-      );
-      itemMarkContainer.addChild(graphics);
-      textContainer.addChild(texts);
-    });
-
-    sections.forEach((section) => {
-      const { content } = section;
-      const ifHighlight = highlightSectionNames.includes(section.name);
-
-      const sectionTheme = ifHighlight ? '#FFFFFF' : colorForName('section', section.name);
-
-      content.forEach((text) => {
-        const { content, rect } = text;
-        const { left, top } = rect;
-        const style = new PIXI.TextStyle({
-          wordWrap: true,
-          wordWrapWidth: rect.width,
-          fontSize: 18,
-          fill: sectionTheme,
-        });
-        const textElement = new PIXI.Text(content, style);
-        textElement.x = left;
-        textElement.y = top;
-        textContainer.addChild(textElement);
-
-        const textBorder = new PIXI.Graphics();
-        textBorder.beginFill(0xaaaaaa, 0.2);
-        textBorder.lineStyle(1, 0x0, 1);
-        textBorder.drawRect(left, top, rect.width, rect.height);
-        textBorder.endFill();
-        textContainer.addChild(textBorder);
-      });
-    });
-    textContainer.visible = textsVisible;
-    return {
-      highlightSectionRects,
-      highlightElementRects,
-    };
-  }, [
-    app,
-    appInitialed,
-    sections,
-    highlightSectionNames,
-    highlightElements,
-    // bgVisible,
-    // textsVisible,
-  ]);
-
-  const onSetBg: CheckboxProps['onChange'] = (e) => {
-    setBgVisible(e.target.checked);
-    if (pixiBgRef.current) {
-      pixiBgRef.current.alpha = e.target.checked ? bgOnAlpha : bgOffAlpha;
-    }
-  };
-
-  const onSetTextsVisible: CheckboxProps['onChange'] = (e) => {
-    setTextsVisible(e.target.checked);
-    textContainer.visible = e.target.checked;
-  };
-
-  let bottomTipA: ReactElement | null = null;
-  if (highlightElementRects.length === 1) {
-    bottomTipA = (
-      <div className="bottom-tip">
-        <div className="bottom-tip-item">Element: {JSON.stringify(highlightElementRects[0])}</div>
-      </div>
-    );
-  } else if (highlightElementRects.length > 1) {
-    bottomTipA = (
-      <div className="bottom-tip">
-        <div className="bottom-tip-item">Element: {JSON.stringify(highlightElementRects)}</div>
-      </div>
-    );
-  }
-
-  let bottomTipB: ReactElement | null = null;
-  if (highlightSectionRects.length === 1) {
-    bottomTipB = (
-      <div className="bottom-tip">
-        <div className="bottom-tip-item">Section: {JSON.stringify(highlightSectionRects[0])}</div>
-      </div>
-    );
-  } else if (highlightSectionRects.length > 1) {
-    bottomTipB = (
-      <div className="bottom-tip">
-        <div className="bottom-tip-item">Sections: {JSON.stringify(highlightSectionRects)}</div>
-      </div>
-    );
-  }
-
-  return (
-    <div className="blackboard">
-      <div className="blackboard-main-content" style={{ width: '100%' }} ref={domRef} />
-      <div className="blackboard-filter">
-        <div className="overlay-control">
-          <Checkbox checked={bgVisible} onChange={onSetBg}>
-            Screenshot
-          </Checkbox>
-          <Checkbox checked={textsVisible} onChange={onSetTextsVisible}>
-            Text Mark
-          </Checkbox>
-        </div>
-      </div>
-      <div className="bottom-tip">
-        {bottomTipA}
-        {bottomTipB}
-      </div>
-
-      {/* {footer} */}
-    </div>
-  );
-};
-
-export default BlackBoard;

package/src/component/color.tsx

@@ -1,34 +0,0 @@
-// https://coolors.co/palettes/popular/#01204e
-const sectionColor = ['#028391'];
-const elementColor = ['#fb6107'];
-const highlightColorForSection = '#01204E';
-const highlightColorForElement = '#F56824'; // @main-orange
-
-function djb2Hash(str?: string): number {
-  if (!str) {
-    console.warn('djb2Hash: empty string');
-    str = 'unnamed';
-  }
-  let hash = 5381;
-  for (let i = 0; i < str.length; i++) {
-    hash = (hash << 5) + hash + str.charCodeAt(i); // hash * 33 + c
-  }
-  return hash >>> 0; // Convert to unsigned 32
-}
-
-export function colorForName(type: 'section' | 'element', name: string): string {
-  const hashNumber = djb2Hash(name);
-  if (type === 'section') {
-    return sectionColor[hashNumber % sectionColor.length];
-  } else {
-    return elementColor[hashNumber % elementColor.length];
-  }
-}
-
-export function highlightColorForType(type: 'section' | 'element'): string {
-  if (type === 'section') {
-    return highlightColorForSection;
-  } else {
-    return highlightColorForElement;
-  }
-}

package/src/component/common.less

@@ -1,21 +0,0 @@
-@main-blue: #06B1AB;
-@sub-blue: #A3D6D2;
-
-@main-orange: #F9483E;
-
-@side-bg: #ECECEC;
-@title-bg: #DDDDDD;
-@border-color: #CCCCCC;
-@heavy-border-color: #888;
-
-@selected-bg: @main-blue;
-@selected-text: #ffffff;
-@hover-bg: @sub-blue;
-@hover-text: #333;
-
-
-@weak-text: #777;
-@weak-bg: #F3F3F3;
-
-@side-horizontal-padding: 10px;
-@side-vertical-spacing: 10px;

package/src/component/detail-panel.less

@@ -1,47 +0,0 @@
-@import './common.less';
-
-@layout-space: 10px;
-
-.detail-panel {
-  display: flex;
-  flex-direction: column;
-  height: 100%;
-  padding: @layout-space;
-  background: #FFF;
-
-  .ant-segmented{
-    padding: 0;
-  }
-
-  .view-switcher{
-    margin-bottom: @layout-space;
-    flex-shrink: 0;
-  }
-
-  .detail-content{
-    box-sizing: border-box;
-    justify-content: center;
-    flex-direction: column;
-    display: flex;
-    flex-grow: 1;
-  }
-
-  .blackboard {
-    margin: 0 auto;
-  }
-
-  .screenshot-item {
-    margin-bottom: @layout-space;
-
-    .screenshot-item-title {
-      // font-weight: bold;
-      margin-bottom: 5px;
-    }
-
-    img {
-      border: 1px solid @heavy-border-color;
-      max-width: 100%;
-    }
-  }
-
-}

package/src/component/detail-panel.tsx

@@ -1,124 +0,0 @@
-'use client';
-import './detail-panel.less';
-import { Segmented, ConfigProvider } from 'antd';
-import { CameraOutlined, FileTextOutlined, ScheduleOutlined } from '@ant-design/icons';
-import { useEffect, useState } from 'react';
-import BlackBoard from './blackboard';
-import { useExecutionDump, useInsightDump } from '@/component/store';
-import { timeStr, filterBase64Value } from '@/utils';
-
-const ScreenshotItem = (props: { time: string; img: string }) => {
-  return (
-    <div className="screenshot-item">
-      <div className="screenshot-item-title">{props.time}</div>
-      <div>
-        <img src={props.img} />
-      </div>
-    </div>
-  );
-};
-
-const VIEW_TYPE_SCREENSHOT = 'screenshot';
-const VIEW_TYPE_JSON = 'json';
-const VIEW_TYPE_BLACKBOARD = 'blackboard';
-
-const DetailPanel = (): JSX.Element => {
-  const dumpId = useInsightDump((store) => store._loadId);
-  const blackboardViewAvailable = Boolean(dumpId);
-  const activeTask = useExecutionDump((store) => store.activeTask);
-  const [preferredViewType, setViewType] = useState(dumpId ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
-
-  const viewType =
-    preferredViewType === VIEW_TYPE_BLACKBOARD && !dumpId ? VIEW_TYPE_SCREENSHOT : preferredViewType;
-
-  let content;
-  if (!activeTask) {
-    content = <div>please select a task</div>;
-  } else if (viewType === VIEW_TYPE_JSON) {
-    content = (
-      <div className="json-content">{filterBase64Value(JSON.stringify(activeTask, undefined, 2))}</div>
-    );
-  } else if (viewType === VIEW_TYPE_BLACKBOARD) {
-    if (dumpId) {
-      content = <BlackBoard key={`${dumpId}`} />;
-    } else {
-      content = <div>invalid view</div>;
-    }
-  } else if (viewType === VIEW_TYPE_SCREENSHOT) {
-    if (activeTask.recorder?.length) {
-      content = (
-        <div>
-          {activeTask.recorder
-            .filter((item) => item.screenshot)
-            .map((item, index) => {
-              const fullTime = timeStr(item.ts);
-              const str = item.timing ? `${fullTime} / ${item.timing}` : fullTime;
-              return <ScreenshotItem key={index} time={str} img={item.screenshot!} />;
-            })}
-        </div>
-      );
-    } else {
-      content = <div>no screenshot</div>; // TODO: pretty error message
-    }
-  }
-
-  useEffect(() => {
-    // hit `Tab` to toggle viewType
-    const handleKeyDown = (e: KeyboardEvent) => {
-      if (e.key === 'Tab') {
-        if (viewType === VIEW_TYPE_BLACKBOARD) {
-          setViewType(VIEW_TYPE_SCREENSHOT);
-        } else if (viewType === VIEW_TYPE_SCREENSHOT) {
-          setViewType(VIEW_TYPE_JSON);
-        } else {
-          setViewType(blackboardViewAvailable ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
-        }
-        e.preventDefault();
-      }
-    };
-
-    document.addEventListener('keydown', handleKeyDown);
-
-    return () => {
-      document.removeEventListener('keydown', handleKeyDown);
-    };
-  });
-
-  const options = [
-    { label: 'Screenshots', value: VIEW_TYPE_SCREENSHOT, icon: <CameraOutlined /> },
-    { label: 'JSON View', value: VIEW_TYPE_JSON, icon: <FileTextOutlined /> },
-  ];
-  if (blackboardViewAvailable) {
-    options.unshift({ label: 'Visualization', value: VIEW_TYPE_BLACKBOARD, icon: <ScheduleOutlined /> });
-  }
-  return (
-    <div className="detail-panel">
-      <div className="view-switcher">
-        <ConfigProvider
-          theme={{
-            components: {
-              Segmented: {
-                itemSelectedBg: '#D7D7D7',
-                itemSelectedColor: '#000000',
-                // itemHoverColor: '#ffffff',
-                // itemHoverBg: '#A3D6D2', // @sub-blue
-                /* 这里是你的组件 token */
-              },
-            },
-          }}
-        >
-          <Segmented
-            options={options}
-            value={viewType}
-            onChange={(value: any) => {
-              setViewType(value);
-            }}
-          />
-        </ConfigProvider>
-      </div>
-      <div className="detail-content">{content}</div>
-    </div>
-  );
-};
-
-export default DetailPanel;

package/src/component/detail-side.less

@@ -1,131 +0,0 @@
-@import './common.less';
-
-@subSpacing: 5px;
-@lightTextColor: #777;
-
-.detail-side {
-  h2 {
-    padding-top: 0;
-    margin-top: 0;
-    margin-bottom: 4px;
-  }
-
-  .ant-tag{
-    margin-top: 2px;
-  }
-
-  .meta-kv {
-    padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
-    .meta {
-      box-sizing: border-box;
-      padding: 2px 0;
-      width: 100%;
-      display: flex;
-      flex-direction: row;
-      line-height: 1.5;
-
-      .meta-key {
-        // font-weight: bold;
-        width:100px;
-        text-align: right;
-        padding-right: 10px;
-        // color: #999;
-      }
-      
-      .meta-value {
-        flex: 1;
-      }
-    }
-  }
-
-  .item-list {
-    padding: @side-vertical-spacing @side-horizontal-padding;
-
-    cursor:default;
-    margin-bottom: 10px;
-
-    .item {
-      padding: 16px @side-horizontal-padding @side-horizontal-padding;
-      transition: .1s;
-      border: 1px solid #DDD;
-      border-radius: @subSpacing;
-      margin-bottom: @side-vertical-spacing;
-      position: relative;
-
-      &.item-lite {
-        border: none;
-        padding: 0;
-      }
-    }
-
-    .item-highlight {
-      color: #FFF;
-      .subtitle {
-        color: #CCC;
-      }
-    }
-    
-    .item-extra {
-      position: absolute;
-      right: @side-horizontal-padding;
-      top: @side-horizontal-padding;
-      color: @lightTextColor;
-    }
-
-    .title-right-padding {
-      padding-right: 15px;
-    }
-  
-    .title {
-      font-size: 18px;
-      font-weight: bold;
-      margin-bottom: @subSpacing;
-
-      .title-tag {
-        display: inline-block;
-        margin-left: 6px;
-        color: @lightTextColor;
-        font-size: 14px;
-        line-height: 18px;
-      }
-    }
-
-    .subtitle {
-      font-weight: normal;
-      font-size: 14px;
-      color: #777;
-    }
-
-    .description {
-      margin-top: @subSpacing;
-    }
-  
-    .description-content {
-      font-size: 14px;
-      margin-top: @side-horizontal-padding;
-      white-space: break-spaces;
-      margin: 0;
-    }
-
-    .element-button:hover {
-      // font-weight: bold;
-      color: #fff;
-      background: @main-orange;
-    }
-
-    .section-button:hover {
-      color: #fff;
-      background: #01204E;
-    }
-  }
-
-  .context {
-    pre {
-      text-wrap: balance;
-    }
-  }
-
-  .item-list-space-up {
-    margin-top: @side-vertical-spacing;
-  }
-}