@midscene/visualizer 0.5.2-beta-20241009023204.0 → 0.5.2

package/dist/es/component/detail-panel.js

@@ -91,7 +91,7 @@ const DetailPanel = () => {
   const options = availableViewTypes.map((type) => {
     if (type === VIEW_TYPE_REPLAY) {
       return {
-        label: "Task Replay",
+        label: "Replay",
         value: type,
         icon: /* @__PURE__ */ jsx(VideoCameraOutlined, {})
       };

package/dist/es/component/player.css

@@ -1,51 +1,97 @@
 .player-container {
-  width: 100%;
-  height: 100%;
+  width: fit-content;
+  max-width: 100%;
+  max-height: 100%;
+  padding: 12px;
+  padding-bottom: 0;
+  background: #434443DD;
+  box-sizing: border-box;
+  border: 1px solid #979797;
+  border-radius: 6px;
+  line-height: 100%;
+  margin: 0 auto;
   display: flex;
   flex-direction: column;
-  justify-content: center;
-  position: relative;
-}
-.player-container .player-top-placeholder {
-  width: 100%;
-  height: 1px;
+  overflow: hidden;
 }
 .player-container .canvas-container {
-  width: 100%;
-  height: 90%;
-  text-align: center;
-  position: relative;
+  flex-grow: 1;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  overflow: hidden;
 }
-.player-container canvas {
-  border: 1px solid #06B1AB;
+.player-container .canvas-container canvas {
   max-width: 100%;
   max-height: 100%;
   box-sizing: border-box;
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
+  display: block;
+  margin: 0 auto;
 }
-.player-container .player-controls {
+.player-container .player-timeline {
+  width: 100%;
+  height: 4px;
+  background: #666;
+  position: relative;
+  margin-top: -2px;
+}
+.player-container .player-timeline .player-timeline-progress {
+  transition-timing-function: linear;
   position: absolute;
-  bottom: 20px;
+  top: 0;
   left: 0;
-  width: 100%;
-  background: rgba(0, 0, 0, 0.7);
-  border-radius: 8px;
-  padding: 20px 15px;
+  height: 4px;
+  background: #06b1ab;
+}
+.player-container .player-controls {
+  margin-top: calc(12px * 1.2);
+  margin-bottom: calc(12px * 1.2);
+  max-width: 100%;
+  overflow: hidden;
   color: #FFF;
-  font-size: 18px;
-  line-height: 1.2;
-  transition: 0.2s;
+  font-size: 16px;
   box-sizing: border-box;
+  display: flex;
+  flex-direction: row;
+  flex-shrink: 0;
+}
+.player-container .player-controls .status-icon {
+  transition: 0.2s;
+  width: 40px;
+  height: 40px;
+  margin-right: 12px;
+  background: #666;
+  border-radius: 6px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  flex-shrink: 0;
+}
+.player-container .player-controls .status-text {
+  flex-grow: 1;
+  overflow: hidden;
+  position: relative;
 }
 .player-container .player-controls .title {
   font-weight: bold;
+  height: 20px;
+  line-height: 20px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
 }
 .player-container .player-controls .subtitle {
-  white-space: nowrap;
-  text-overflow: ellipsis;
-  overflow: hidden;
+  height: 20px;
+  line-height: 20px;
   width: 100%;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  position: absolute;
+  top: 18px;
+  left: 0;
 }

package/dist/es/component/player.js

@@ -40,6 +40,11 @@ import * as PIXI from "pixi.js";
 import { useEffect, useMemo, useRef, useState } from "react";
 import "./player.css";
 import { mouseLoading, mousePointer } from "../utils";
+import {
+  LoadingOutlined,
+  PlayCircleOutlined
+} from "@ant-design/icons";
+import { ConfigProvider, Spin } from "antd";
 import { rectMarkForItem } from "./blackboard";
 import { useExecutionDump } from "./store";
 const canvasPaddingLeft = 0;
@@ -49,7 +54,7 @@ const cubicBezier = (t, p0, p1, p2, p3) => {
   return p0 * t2 * t2 * t2 + 3 * p1 * t * t2 * t2 + 3 * p2 * t * t * t2 + p3 * t * t * t;
 };
 const cubicImage = (t) => {
-  return cubicBezier(t, 0, 0.69, 0.43, 1);
+  return linear(t);
 };
 const cubicInsightElement = (t) => {
   return cubicBezier(t, 0, 0.5, 0.5, 1);
@@ -63,6 +68,38 @@ const linear = (t) => {
 const sleep = (ms) => {
   return new Promise((resolve) => setTimeout(resolve, ms));
 };
+const ERROR_FRAME_CANCEL = "frame cancel";
+const frameKit = () => {
+  let cancelFlag = false;
+  return {
+    frame: (callback) => {
+      if (cancelFlag) {
+        throw new Error(ERROR_FRAME_CANCEL);
+      }
+      requestAnimationFrame(() => {
+        if (cancelFlag) {
+          throw new Error(ERROR_FRAME_CANCEL);
+        }
+        callback(performance.now());
+      });
+    },
+    timeout: (callback, ms) => {
+      if (cancelFlag) {
+        throw new Error(ERROR_FRAME_CANCEL);
+      }
+      setTimeout(() => {
+        if (cancelFlag) {
+          throw new Error(ERROR_FRAME_CANCEL);
+        }
+        callback();
+      }, ms);
+    },
+    cancel: () => {
+      console.log("set frame cancel");
+      cancelFlag = true;
+    }
+  };
+};
 const singleElementFadeInDuration = 80;
 const LAYER_ORDER_IMG = 0;
 const LAYER_ORDER_INSIGHT = 1;
@@ -87,6 +124,7 @@ const Player = () => {
   const imgSpriteMap = useRef(/* @__PURE__ */ new Map());
   const pointerSprite = useRef(null);
   const spinningPointerSprite = useRef(null);
+  const [replayMark, setReplayMark] = useState(0);
   const windowContentContainer = useMemo(() => {
     const container = new PIXI.Container();
     return container;
@@ -105,6 +143,7 @@ const Player = () => {
       top: imageHeight / 2
     }
   };
+  const [animationProgress, setAnimationProgress] = useState(-1);
   const cancelFlag = useRef(false);
   useEffect(() => {
     cancelFlag.current = false;
@@ -112,14 +151,6 @@ const Player = () => {
       cancelFlag.current = true;
     };
   }, []);
-  const frame = (callback) => {
-    requestAnimationFrame(() => {
-      if (cancelFlag.current) {
-        return;
-      }
-      callback(performance.now());
-    });
-  };
   const cameraState = useRef(__spreadValues({}, basicCameraState));
   const preloadImage = (img) => __async(void 0, null, function* () {
     if (imgSpriteMap.current.has(img))
@@ -151,7 +182,7 @@ const Player = () => {
     sprite.zIndex = LAYER_ORDER_IMG;
     windowContentContainer.addChild(sprite);
   });
-  const spinningPointer = () => __async(void 0, null, function* () {
+  const spinningPointer = (frame) => {
     var _a2, _b;
     if (!spinningPointerSprite.current) {
       spinningPointerSprite.current = PIXI.Sprite.from(mouseLoading);
@@ -186,7 +217,7 @@ const Player = () => {
       isCancelled = true;
     };
     return stopFn;
-  });
+  };
   const updatePointer = (img, x, y) => __async(void 0, null, function* () {
     var _a2, _b;
     if (!imgSpriteMap.current.has(img)) {
@@ -239,7 +270,7 @@ const Player = () => {
       pointer.scale.set(1 / newScale);
     }
   };
-  const cameraAnimation = (targetState, duration) => __async(void 0, null, function* () {
+  const cameraAnimation = (targetState, duration, frame) => __async(void 0, null, function* () {
     const currentState = __spreadValues({}, cameraState.current);
     const startLeft = currentState.left;
     const startTop = currentState.top;
@@ -292,7 +323,7 @@ const Player = () => {
       frame(animate);
     });
   });
-  const fadeInGraphics = (graphics, duration, targetAlpha = 1) => {
+  const fadeInGraphics = (graphics, duration, frame, targetAlpha = 1) => {
     return new Promise((resolve) => {
       const startTime = performance.now();
       const animate = (currentTime) => {
@@ -308,13 +339,14 @@ const Player = () => {
       frame(animate);
     });
   };
-  const fadeOutItem = (graphics, duration) => __async(void 0, null, function* () {
-    return fadeInGraphics(graphics, duration, 0);
+  const fadeOutItem = (graphics, duration, frame) => __async(void 0, null, function* () {
+    return fadeInGraphics(graphics, duration, frame, 0);
   });
-  const insightElementsAnimation = (elements, highlightElements, duration) => __async(void 0, null, function* () {
+  const insightElementsAnimation = (elements, highlightElements, duration, frame) => __async(void 0, null, function* () {
     insightMarkContainer.removeChildren();
     const elementsToAdd = [...elements];
     const totalLength = elementsToAdd.length;
+    let childrenCount = 0;
     yield new Promise((resolve) => {
       const startTime = performance.now();
       const animate = (currentTime) => {
@@ -323,7 +355,7 @@ const Player = () => {
           Math.min(elapsedTime / duration, 1)
         );
         const elementsToAddNow = Math.floor(progress * totalLength);
-        while (insightMarkContainer.children.length < elementsToAddNow) {
+        while (childrenCount < elementsToAddNow) {
           const randomIndex = Math.floor(Math.random() * elementsToAdd.length);
           const element = elementsToAdd.splice(randomIndex, 1)[0];
           if (element) {
@@ -334,7 +366,12 @@ const Player = () => {
             );
             insightMarkGraphic.alpha = 0;
             insightMarkContainer.addChild(insightMarkGraphic);
-            fadeInGraphics(insightMarkGraphic, singleElementFadeInDuration);
+            childrenCount++;
+            fadeInGraphics(
+              insightMarkGraphic,
+              singleElementFadeInDuration,
+              frame
+            );
           }
         }
         if (elapsedTime < duration) {
@@ -368,30 +405,57 @@ const Player = () => {
       frame(animate);
     });
   });
-  useEffect(() => {
+  const init = () => __async(void 0, null, function* () {
+    if (!divContainerRef.current || !scripts)
+      return;
+    yield app.init({
+      width: canvasWidth,
+      height: canvasHeight,
+      background: 16053492
+    });
+    divContainerRef.current.appendChild(app.canvas);
+    windowContentContainer.x = 0;
+    windowContentContainer.y = 0;
+    app.stage.addChild(windowContentContainer);
+    insightMarkContainer.x = 0;
+    insightMarkContainer.y = 0;
+    windowContentContainer.addChild(insightMarkContainer);
+  });
+  const play = () => {
     let cancelFn;
     Promise.resolve(
       (() => __async(void 0, null, function* () {
-        if (!divContainerRef.current || !scripts || !app)
-          return;
-        yield app.init({
-          width: canvasWidth,
-          height: canvasHeight,
-          background: 16053492
-        });
-        divContainerRef.current.appendChild(app.canvas);
+        if (!app) {
+          throw new Error("app is not initialized");
+        }
+        if (!scripts) {
+          throw new Error("scripts is required");
+        }
+        const { frame, cancel, timeout } = frameKit();
+        cancelFn = cancel;
         const allImages = scripts.filter((item) => !!item.img).map((item) => item.img);
         yield Promise.all([...allImages, mouseLoading].map(preloadImage));
-        windowContentContainer.x = 0;
-        windowContentContainer.y = 0;
-        app.stage.addChild(windowContentContainer);
-        insightMarkContainer.x = 0;
-        insightMarkContainer.y = 0;
-        windowContentContainer.addChild(insightMarkContainer);
+        insightMarkContainer.removeChildren();
         yield updatePointer(mousePointer, imageWidth / 2, imageHeight / 2);
         yield repaintImage();
-        yield updateCamera(cameraState.current);
-        for (const item of scripts) {
+        yield updateCamera(__spreadValues({}, basicCameraState));
+        const totalDuration = scripts.reduce((acc, item) => {
+          return acc + item.duration + (item.insightCameraDuration || 0);
+        }, 0);
+        const progressUpdateInterval = 300;
+        const startTime = performance.now();
+        setAnimationProgress(0);
+        const updateProgress = () => {
+          const progress = Math.min(
+            (performance.now() - startTime) / totalDuration,
+            1
+          );
+          setAnimationProgress(progress);
+          return timeout(updateProgress, progressUpdateInterval);
+        };
+        frame(updateProgress);
+        for (const index in scripts) {
+          const item = scripts[index];
           setTitleText(item.title || "");
           setSubTitleText(item.subTitle || "");
           if (item.type === "sleep") {
@@ -407,13 +471,21 @@ const Player = () => {
             yield insightElementsAnimation(
               elements,
               highlightElements,
-              item.duration
+              item.duration,
+              frame
             );
             if (item.camera) {
-              yield cameraAnimation(item.camera, item.duration);
+              if (!item.insightCameraDuration) {
+                throw new Error("insightCameraDuration is required");
+              }
+              yield cameraAnimation(
+                item.camera,
+                item.insightCameraDuration,
+                frame
+              );
             }
           } else if (item.type === "clear-insight") {
-            yield fadeOutItem(insightMarkContainer, item.duration);
+            yield fadeOutItem(insightMarkContainer, item.duration, frame);
             insightMarkContainer.removeChildren();
             insightMarkContainer.alpha = 1;
           } else if (item.type === "img") {
@@ -422,7 +494,7 @@ const Player = () => {
               yield repaintImage();
             }
             if (item.camera) {
-              yield cameraAnimation(item.camera, item.duration);
+              yield cameraAnimation(item.camera, item.duration, frame);
             } else {
               yield sleep(item.duration);
             }
@@ -432,29 +504,98 @@ const Player = () => {
             }
             yield updatePointer(item.img);
           } else if (item.type === "spinning-pointer") {
-            const cancelFn2 = yield spinningPointer();
+            const stop = spinningPointer(frame);
             yield sleep(item.duration);
-            cancelFn2();
+            stop();
           }
         }
+      }))().catch((e) => {
+        console.error("player error", e);
+      })
+    );
+    return () => {
+      cancelFn == null ? void 0 : cancelFn();
+    };
+  };
+  useEffect(() => {
+    Promise.resolve(
+      (() => __async(void 0, null, function* () {
+        yield init();
+        setReplayMark(Date.now());
       }))()
     );
     return () => {
       try {
-        cancelFn == null ? void 0 : cancelFn();
         app.destroy(true, { children: true, texture: true });
       } catch (e) {
         console.warn("destroy failed", e);
       }
     };
   }, []);
-  const showControl = !!titleText;
+  useEffect(() => {
+    if (replayMark) {
+      return play();
+    }
+  }, [replayMark]);
+  const progressString = Math.round(animationProgress * 100);
+  const transitionStyle = animationProgress === 0 ? "none" : "0.3s";
+  const [mouseOverStatusIcon, setMouseOverStatusIcon] = useState(false);
+  let statusIconElement;
+  const statusStyle = {};
+  let statusOnClick = () => {
+  };
+  if (animationProgress < 1) {
+    statusIconElement = /* @__PURE__ */ jsx(Spin, { indicator: /* @__PURE__ */ jsx(LoadingOutlined, { spin: true }), size: "default" });
+  } else if (mouseOverStatusIcon) {
+    statusIconElement = /* @__PURE__ */ jsx(Spin, { indicator: /* @__PURE__ */ jsx(PlayCircleOutlined, {}), size: "default" });
+    statusStyle.cursor = "pointer";
+    statusStyle.background = "#888";
+    statusOnClick = () => setReplayMark(Date.now());
+  } else {
+    statusIconElement = // <Spin indicator={<CheckCircleOutlined />} size="default" />
+    /* @__PURE__ */ jsx(Spin, { indicator: /* @__PURE__ */ jsx(PlayCircleOutlined, {}), size: "default" });
+  }
   return /* @__PURE__ */ jsxs("div", { className: "player-container", children: [
-    /* @__PURE__ */ jsx("div", { className: "player-top-placeholder", children: " " }),
     /* @__PURE__ */ jsx("div", { className: "canvas-container", ref: divContainerRef }),
-    /* @__PURE__ */ jsxs("div", { className: "player-controls", style: { opacity: showControl ? 1 : 0 }, children: [
-      /* @__PURE__ */ jsx("div", { className: "title", children: titleText }),
-      /* @__PURE__ */ jsx("div", { className: "subtitle", children: subTitleText })
+    /* @__PURE__ */ jsx("div", { className: "player-timeline", children: /* @__PURE__ */ jsx(
+      "div",
+      {
+        className: "player-timeline-progress",
+        style: {
+          width: `${progressString}%`,
+          transition: transitionStyle
+        }
+      }
+    ) }),
+    /* @__PURE__ */ jsxs("div", { className: "player-controls", children: [
+      /* @__PURE__ */ jsx(
+        "div",
+        {
+          className: "status-icon",
+          onMouseEnter: () => setMouseOverStatusIcon(true),
+          onMouseLeave: () => setMouseOverStatusIcon(false),
+          style: statusStyle,
+          onClick: statusOnClick,
+          children: /* @__PURE__ */ jsx(
+            ConfigProvider,
+            {
+              theme: {
+                components: {
+                  Spin: {
+                    dotSize: 24,
+                    colorPrimary: "rgb(6,177,171)"
+                  }
+                }
+              },
+              children: statusIconElement
+            }
+          )
+        }
+      ),
+      /* @__PURE__ */ jsxs("div", { className: "status-text", children: [
+        /* @__PURE__ */ jsx("div", { className: "title", children: titleText }),
+        /* @__PURE__ */ jsx("div", { className: "subtitle", children: subTitleText })
+      ] })
     ] })
   ] });
 };

package/dist/es/component/{player-scripts.js → replay-scripts.js}

@@ -33,7 +33,7 @@ const cameraStateForRect = (rect, imageWidth, imageHeight) => {
   } else {
     rectWidthOnPage = rect.height / imageHeight * imageWidth;
   }
-  const cameraPaddingRatio = rectWidthOnPage < 50 ? 0.3 : 0.2;
+  const cameraPaddingRatio = rectWidthOnPage > 400 ? 0.1 : rectWidthOnPage > 50 ? 0.2 : 0.3;
   const cameraWidth = Math.min(
     imageWidth,
     rectWidthOnPage + imageWidth * cameraPaddingRatio * 2
@@ -103,6 +103,7 @@ const generateAnimationScripts = (execution, imageWidth, imageHeight) => {
   let currentCameraState = fullPageCameraState;
   let insightOnTop = false;
   const taskCount = execution.tasks.length;
+  let initSubTitle = "";
   execution.tasks.forEach((task, index) => {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
     if (task.type === "Planning") {
@@ -111,10 +112,12 @@ const generateAnimationScripts = (execution, imageWidth, imageHeight) => {
         scripts.push({
           type: "img",
           img: (_b = (_a = planningTask.recorder) == null ? void 0 : _a[0]) == null ? void 0 : _b.screenshot,
+          camera: fullPageCameraState,
           duration: stillDuration,
           title: typeStr(task),
           subTitle: paramStr(task)
         });
+        initSubTitle = paramStr(task);
       }
     } else if (task.type === "Insight") {
       const insightTask = task;
@@ -141,6 +144,7 @@ const generateAnimationScripts = (execution, imageWidth, imageHeight) => {
           insightDump,
           camera: currentCameraState === fullPageCameraState || !insightCameraState ? void 0 : mergeTwoCameraState(currentCameraState, insightCameraState),
           duration: insightContentLength > 20 ? locateDuration : locateDuration * 0.5,
+          insightCameraDuration: locateDuration,
           title,
           subTitle
         });
@@ -201,14 +205,12 @@ const generateAnimationScripts = (execution, imageWidth, imageHeight) => {
     }
   });
   scripts.push({
+    title: "Done",
+    subTitle: initSubTitle,
     type: "img",
     duration: stillDuration * 0.5,
     camera: fullPageCameraState
   });
-  scripts.push({
-    type: "sleep",
-    duration: stillDuration
-  });
   return scripts;
 };
 export {

package/dist/es/component/store.js

@@ -18,7 +18,7 @@ var __spreadValues = (a, b) => {
 };
 var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
 import * as Z from "zustand";
-import { generateAnimationScripts } from "./player-scripts";
+import { generateAnimationScripts } from "./replay-scripts";
 const { create } = Z;
 const useBlackboardPreference = create((set) => ({
   bgVisible: true,

package/dist/es/index.css

@@ -125,7 +125,7 @@ footer.mt-8 {
   text-align: center;
   margin-top: 10px;
 }
-.main-right .main-canvas-container {
+.main-right .main-content-container {
   flex-grow: 1;
   height: 100%;
   background: #ffffff;

package/dist/es/index.js

@@ -148,7 +148,7 @@ function Visualizer(props) {
                 autoSaveId: "page-detail-layout-v2",
                 direction: "horizontal",
                 children: [
-                  /* @__PURE__ */ jsx(Panel, { defaultSize: 75, maxSize: 95, children: /* @__PURE__ */ jsx("div", { className: "main-canvas-container", children: /* @__PURE__ */ jsx(DetailPanel, {}) }) }),
+                  /* @__PURE__ */ jsx(Panel, { defaultSize: 75, maxSize: 95, children: /* @__PURE__ */ jsx("div", { className: "main-content-container", children: /* @__PURE__ */ jsx(DetailPanel, {}) }) }),
                   /* @__PURE__ */ jsx(PanelResizeHandle, {}),
                   /* @__PURE__ */ jsx(Panel, { maxSize: 95, children: /* @__PURE__ */ jsx("div", { className: "main-side", children: /* @__PURE__ */ jsx(DetailSide, {}) }) })
                 ]