@midscene/visualizer 0.0.1

package/src/component/sidebar.tsx

@@ -0,0 +1,205 @@
+import './sidebar.less';
+import { useEffect } from 'react';
+import {
+  ArrowRightOutlined,
+  CheckOutlined,
+  ClockCircleOutlined,
+  CloseOutlined,
+  LogoutOutlined,
+  MinusOutlined,
+} from '@ant-design/icons';
+import { ExecutionTask } from '@midscene/core';
+import { Button } from 'antd';
+import PanelTitle from './panel-title';
+import { timeCostStrElement } from './misc';
+import logo from './assets/logo-plain2.svg';
+import { useAllCurrentTasks, useExecutionDump } from '@/component/store';
+import { typeStr } from '@/utils';
+
+const SideItem = (props: {
+  task: ExecutionTask;
+  selected?: boolean;
+  onClick?: () => void;
+  onItemHover?: (task: ExecutionTask | null, x?: number, y?: number) => any;
+}): JSX.Element => {
+  const { task, onClick, selected } = props;
+
+  const selectedClass = selected ? 'selected' : '';
+  let statusIcon = <MinusOutlined />;
+  if (task.status === 'success') {
+    statusIcon = <CheckOutlined />;
+  } else if (task.status === 'fail') {
+    statusIcon = <CloseOutlined />;
+  } else if (task.status === 'pending') {
+    statusIcon = <ClockCircleOutlined />;
+  } else if (task.status === 'cancelled') {
+    statusIcon = <LogoutOutlined />;
+  } else if (task.status === 'running') {
+    statusIcon = <ArrowRightOutlined />;
+  }
+
+  let statusText: JSX.Element | string = task.status;
+  if (task.timing?.cost) {
+    statusText = timeCostStrElement(task.timing.cost);
+  }
+
+  const contentRow =
+    task.type === 'Planning' ? <div className="side-item-content">{task.param?.userPrompt} </div> : null;
+  // add hover listener
+  return (
+    <div
+      className={`side-item ${selectedClass}`}
+      onClick={onClick}
+      // collect x,y (refer to the body) for hover preview
+      onMouseEnter={(event) => {
+        const rect = event.currentTarget.getBoundingClientRect();
+        const x = rect.left + rect.width;
+        const y = rect.top;
+        props.onItemHover?.(task, x, y);
+      }}
+      onMouseLeave={() => {
+        props.onItemHover?.(null);
+      }}
+    >
+      {' '}
+      <div className={`side-item-name`}>
+        <span className={`status-icon status-icon-${task.status}`}>{statusIcon}</span>
+        <div className="title">{typeStr(task)}</div>
+        <div className="status-text">{statusText}</div>
+      </div>
+      {contentRow}
+    </div>
+  );
+};
+
+const Sidebar = (): JSX.Element => {
+  const groupedDumps = useExecutionDump((store) => store.dump);
+  const setActiveTask = useExecutionDump((store) => store.setActiveTask);
+  const activeTask = useExecutionDump((store) => store.activeTask);
+  const setHoverTask = useExecutionDump((store) => store.setHoverTask);
+  const setHoverPreviewConfig = useExecutionDump((store) => store.setHoverPreviewConfig);
+  // const selectedTaskIndex = useExecutionDump((store) => store.selectedTaskIndex);
+  // const setSelectedTaskIndex = useExecutionDump((store) => store.setSelectedTaskIndex);
+  const reset = useExecutionDump((store) => store.reset);
+
+  const allTasks = useAllCurrentTasks();
+  const currentSelectedIndex = allTasks?.findIndex((task) => task === activeTask);
+
+  useEffect(() => {
+    // all tasks
+    const handleKeyDown = (e: KeyboardEvent) => {
+      if (!allTasks?.length || allTasks?.length <= 1) {
+        return;
+      }
+      // should be command / ctrl + arrow
+      if (e.key === 'ArrowUp' && (e.metaKey || e.ctrlKey)) {
+        e.preventDefault();
+        const nextIndex = currentSelectedIndex - 1;
+        if (nextIndex < 0) {
+          return;
+        }
+        const nextTask = allTasks[nextIndex];
+        setActiveTask(nextTask);
+      } else if (e.key === 'ArrowDown' && (e.metaKey || e.ctrlKey)) {
+        e.preventDefault();
+        const nextIndex = currentSelectedIndex + 1;
+        if (nextIndex >= allTasks.length) {
+          return;
+        }
+        const nextTask = allTasks[nextIndex];
+        setActiveTask(nextTask);
+      }
+    };
+    document.addEventListener('keydown', handleKeyDown);
+    return () => {
+      document.removeEventListener('keydown', handleKeyDown);
+    };
+  }, [currentSelectedIndex, allTasks, setActiveTask]);
+
+  const sideList = groupedDumps?.length ? (
+    groupedDumps.map((group, groupIndex) => {
+      const executions = group.executions.map((execution, indexOfExecution) => {
+        const { tasks } = execution;
+        const taskList = tasks.map((task, index) => {
+          return (
+            <SideItem
+              key={index}
+              task={task}
+              selected={task === activeTask}
+              onClick={() => {
+                setActiveTask(task);
+              }}
+              onItemHover={(hoverTask, x, y) => {
+                if (hoverTask && x && y) {
+                  setHoverPreviewConfig({ x, y });
+                  setHoverTask(hoverTask);
+                } else {
+                  setHoverPreviewConfig(null);
+                  setHoverTask(null);
+                }
+              }}
+            />
+          );
+        });
+        let seperator: JSX.Element;
+        switch (indexOfExecution) {
+          case 0:
+            seperator = <div className="side-seperator side-seperator-space-up" />;
+            break;
+          // case group.executions.length - 1:
+          //   seperator = <div className="side-seperator side-seperator-space-down" />;
+          //   break;
+          default:
+            seperator = (
+              <div className="side-seperator side-seperator-line side-seperator-space-up side-seperator-space-down" />
+            );
+            break;
+        }
+        return (
+          <div key={indexOfExecution}>
+            {seperator}
+            <div className="side-sub-title">{execution.name}</div>
+            {taskList}
+          </div>
+        );
+      });
+      return (
+        <div key={groupIndex}>
+          <PanelTitle title={group.groupName} />
+          {executions}
+        </div>
+      );
+    })
+  ) : (
+    <span>no tasks</span>
+  );
+
+  return (
+    <div className="side-bar">
+      <div className="top-controls">
+        <div className="brand" onClick={reset}>
+          <img
+            src={logo}
+            alt="Logo"
+            style={{ width: 70, height: 70, margin: 'auto' }}
+            onClick={() => {
+              location.reload();
+            }}
+          />
+        </div>
+        <div className="task-list">{sideList}</div>
+        <div className="side-seperator side-seperator-line side-seperator-space-up" />
+        <div className="task-meta-section">
+          <div className="task-meta">use Command + ⬆︎ / ⬇︎ to switch</div>
+        </div>
+      </div>
+      <div className="bottom-controls">
+        <Button onClick={reset} type="text" className="unload_btn">
+          Unload
+        </Button>
+      </div>
+    </div>
+  );
+};
+
+export default Sidebar;

package/src/component/store.tsx

@@ -0,0 +1,151 @@
+import { create } from 'zustand';
+import {
+  InsightDump,
+  BaseElement,
+  ExecutionTaskInsightLocate,
+  ExecutionTask,
+  GroupedActionDump,
+} from '../../../midscene/dist/types';
+
+export const useBlackboardPreference = create<{
+  bgVisible: boolean;
+  textsVisible: boolean;
+  setBgVisible: (visible: boolean) => void;
+  setTextsVisible: (visible: boolean) => void;
+}>((set) => ({
+  bgVisible: false,
+  textsVisible: true,
+  setBgVisible: (visible: boolean) => {
+    set({ bgVisible: visible });
+  },
+  setTextsVisible: (visible: boolean) => {
+    set({ textsVisible: visible });
+  },
+}));
+
+export const useExecutionDump = create<{
+  dump: GroupedActionDump[] | null;
+  setGroupedDump: (dump: GroupedActionDump[]) => void;
+  activeTask: ExecutionTask | null;
+  setActiveTask: (task: ExecutionTask) => void;
+  hoverTask: ExecutionTask | null;
+  setHoverTask: (task: ExecutionTask | null) => void;
+  hoverPreviewConfig: { x: number; y: number } | null;
+  setHoverPreviewConfig: (config: { x: number; y: number } | null) => void;
+  reset: () => void;
+}>((set) => {
+  const initData = {
+    dump: null,
+    activeTask: null,
+    hoverTask: null,
+    hoverPreviewConfig: null,
+  };
+
+  const syncToInsightDump = (dump: InsightDump) => {
+    const { loadData } = useInsightDump.getState();
+    loadData(dump);
+  };
+
+  const resetInsightDump = () => {
+    const { reset } = useInsightDump.getState();
+    reset();
+  };
+
+  return {
+    ...initData,
+    setGroupedDump: (dump: GroupedActionDump[]) => {
+      console.log('will set ExecutionDump', dump);
+      set({
+        dump,
+      });
+
+      // set the first one as selected
+      for (const item of dump) {
+        if (item.executions.length > 0 && item.executions[0].tasks.length > 0) {
+          set({ activeTask: item.executions[0].tasks[0] });
+          break;
+        }
+      }
+    },
+    setActiveTask(task: ExecutionTask) {
+      set({ activeTask: task });
+      if ((task as ExecutionTaskInsightLocate).log?.dump?.matchedElement) {
+        syncToInsightDump((task as ExecutionTaskInsightLocate).log!.dump!);
+      } else {
+        resetInsightDump();
+      }
+    },
+    setHoverTask(task: ExecutionTask | null) {
+      set({ hoverTask: task });
+    },
+    setHoverPreviewConfig(config: { x: number; y: number } | null) {
+      if (config) {
+        set({
+          hoverPreviewConfig: {
+            x: Math.floor(config.x),
+            y: Math.floor(config.y),
+          },
+        });
+      } else {
+        set({ hoverPreviewConfig: null });
+      }
+    },
+    reset: () => {
+      set(initData);
+      resetInsightDump();
+    },
+  };
+});
+
+export const useAllCurrentTasks = (): ExecutionTask[] => {
+  const groupedDumps = useExecutionDump((store) => store.dump);
+
+  const allTasks =
+    groupedDumps?.reduce<ExecutionTask[]>((acc, group) => {
+      const tasksInside = group.executions.reduce<ExecutionTask[]>(
+        (acc2, execution) => acc2.concat(execution.tasks),
+        [],
+      );
+
+      return acc.concat(tasksInside);
+    }, []) || [];
+
+  return allTasks;
+};
+
+export const useInsightDump = create<{
+  _loadId: number;
+  data: InsightDump | null;
+  highlightSectionNames: string[];
+  setHighlightSectionNames: (sections: string[]) => void;
+  highlightElements: BaseElement[];
+  setHighlightElements: (elements: BaseElement[]) => void;
+  loadData: (data: InsightDump) => void;
+  reset: () => void;
+}>((set) => {
+  let loadId = 0;
+  const initData = { _loadId: 0, highlightSectionNames: [], highlightElements: [], data: null };
+
+  return {
+    ...initData,
+    loadData: (data: InsightDump) => {
+      // console.log('will load dump data');
+      // console.log(data);
+      set({
+        _loadId: ++loadId,
+        data,
+        highlightSectionNames: [],
+        highlightElements: [],
+      });
+    },
+    setHighlightSectionNames: (sections: string[]) => {
+      set({ highlightSectionNames: sections });
+    },
+    setHighlightElements: (elements: BaseElement[]) => {
+      set({ highlightElements: elements });
+    },
+    reset: () => {
+      set(initData);
+    },
+  };
+});

package/src/component/timeline.less

@@ -0,0 +1,25 @@
+@import './common.less';
+
+@base-height: 110px;
+
+.timeline-wrapper {
+  flex-basis: @base-height;
+  flex-grow: 0;
+  flex-shrink: 0;
+
+  width: 100%;
+  height: 100%;
+  border-bottom: 1px solid @border-color;
+  position: relative;
+  box-sizing: border-box;
+
+  .timeline-canvas-wrapper {
+    width: 100%;
+    height: 100%;  
+  }
+
+  canvas {
+    width: 100%;
+    height: 100%;
+  }
+}