@godscene/visualizer 1.7.11

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,9 @@
+## Documentation
+
+An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language.
+
+See https://midscenejs.com/ for details.
+
+## License
+
+Midscene is MIT licensed.

package/dist/es/component/blackboard/highlights.mjs

@@ -0,0 +1,47 @@
+function formatCenterKey(center) {
+    return `${center[0]}:${center[1]}`;
+}
+function formatRectKey(rect) {
+    return `${rect.left}:${rect.top}:${rect.width}:${rect.height}`;
+}
+function getElementLabel(element) {
+    if ('content' in element && element.content) return element.content;
+    if ("description" in element && element.description) return element.description;
+}
+function normalizeBlackboardHighlights(elements) {
+    if (!(null == elements ? void 0 : elements.length)) return [];
+    const deduped = new Map();
+    elements.forEach((element, index)=>{
+        if (!(null == element ? void 0 : element.rect) || !(null == element ? void 0 : element.center)) return;
+        const label = getElementLabel(element);
+        const dedupeKey = [
+            'id' in element ? element.id : '',
+            label || '',
+            formatCenterKey(element.center),
+            formatRectKey(element.rect)
+        ].join('|');
+        if (!deduped.has(dedupeKey)) deduped.set(dedupeKey, {
+            key: 'id' in element && element.id || `${dedupeKey || 'highlight'}-${index}`,
+            label,
+            center: element.center,
+            rect: element.rect
+        });
+    });
+    return Array.from(deduped.values());
+}
+function roundRect(rect) {
+    return {
+        left: Math.round(rect.left),
+        top: Math.round(rect.top),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height)
+    };
+}
+function formatBlackboardHighlightSummary(highlight) {
+    const center = `[${Math.round(highlight.center[0])}, ${Math.round(highlight.center[1])}]`;
+    const rect = roundRect(highlight.rect);
+    const rectText = `rect=${JSON.stringify(rect)}`;
+    if (highlight.label) return `${highlight.label} center=${center}, ${rectText}`;
+    return `center=${center}, ${rectText}`;
+}
+export { formatBlackboardHighlightSummary, normalizeBlackboardHighlights };

package/dist/es/component/blackboard/index.css

@@ -0,0 +1,118 @@
+.blackboard .footer {
+  color: #aaa;
+}
+
+.blackboard ul {
+  padding-left: 0;
+}
+
+.blackboard li {
+  list-style: none;
+}
+
+.blackboard .bottom-tip {
+  height: 30px;
+}
+
+.blackboard .bottom-tip-item {
+  color: #aaa;
+  text-overflow: ellipsis;
+  word-wrap: break-word;
+  max-width: 500px;
+}
+
+.blackboard-main-content {
+  position: relative;
+  overflow: hidden;
+}
+
+.blackboard-screenshot {
+  box-sizing: border-box;
+  border: 1px solid #888;
+  width: 100%;
+  display: block;
+}
+
+.blackboard-overlay {
+  pointer-events: none;
+  width: 100%;
+  position: absolute;
+  top: 0;
+  left: 0;
+}
+
+.blackboard-rect {
+  box-sizing: content-box;
+  pointer-events: none;
+  z-index: 1;
+  position: absolute;
+}
+
+.blackboard-rect-label {
+  white-space: nowrap;
+  padding: 1px 4px;
+  font-size: 14px;
+  font-weight: 600;
+  line-height: 1.4;
+  position: absolute;
+  bottom: 100%;
+  left: 0;
+}
+
+.blackboard-rect-search {
+  border: calc(1px * var(--ui-scale, 1)) solid #028391;
+  background: rgba(2, 131, 145, .15);
+}
+
+.blackboard-rect-search .blackboard-rect-label {
+  color: #028391;
+}
+
+.blackboard-rect-highlight {
+  border: calc(1px * var(--ui-scale, 1)) solid #fd5907;
+  background: rgba(253, 89, 7, .15);
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+}
+
+.blackboard-rect-highlight .blackboard-rect-label {
+  color: #000;
+}
+
+.blackboard-point {
+  width: calc(20px * var(--ui-scale, 1));
+  height: calc(20px * var(--ui-scale, 1));
+  margin-left: calc(-10px * var(--ui-scale, 1));
+  margin-top: calc(-10px * var(--ui-scale, 1));
+  border: calc(2px * var(--ui-scale, 1)) solid #fd5907;
+  box-shadow: 0 0 calc(6px * var(--ui-scale, 1)) rgba(253, 89, 7, .5);
+  pointer-events: none;
+  z-index: 2;
+  background: rgba(253, 89, 7, .4);
+  border-radius: 50%;
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+  position: absolute;
+}
+
+@keyframes blackboard-pulse {
+  0%, 100% {
+    opacity: .4;
+  }
+
+  50% {
+    opacity: 1;
+    box-shadow: 0 0 12px rgba(253, 89, 7, .5);
+  }
+}
+
+[data-theme="dark"] .blackboard .footer, [data-theme="dark"] .blackboard .bottom-tip-item {
+  color: rgba(255, 255, 255, .45);
+}
+
+[data-theme="dark"] .blackboard-screenshot {
+  border-color: rgba(255, 255, 255, .12);
+}
+
+[data-theme="dark"] .blackboard-rect-highlight .blackboard-rect-label {
+  color: #fff;
+}
+

package/dist/es/component/blackboard/index.mjs

@@ -0,0 +1,122 @@
+"use client";
+import { jsx, jsxs } from "react/jsx-runtime";
+import react from "react";
+import { getCenterHighlightBox } from "../../utils/highlight-element.mjs";
+import { normalizeBlackboardHighlights } from "./highlights.mjs";
+import "./index.css";
+const Blackboard = (props)=>{
+    var _props_uiContext;
+    const highlightElements = props.highlightElements || [];
+    const highlightRect = props.highlightRect;
+    if (!(null == (_props_uiContext = props.uiContext) ? void 0 : _props_uiContext.shotSize)) return /*#__PURE__*/ jsx("div", {
+        className: "blackboard",
+        children: /*#__PURE__*/ jsx("div", {
+            className: "blackboard-main-content",
+            style: {
+                padding: '20px'
+            },
+            children: "No UI context available"
+        })
+    });
+    const context = props.uiContext;
+    const { shotSize, screenshot } = context;
+    const screenWidth = shotSize.width;
+    const screenHeight = shotSize.height;
+    const highlightOverlays = react.useMemo(()=>normalizeBlackboardHighlights(highlightElements), [
+        highlightElements
+    ]);
+    const screenshotBase64 = react.useMemo(()=>{
+        if (!screenshot) return '';
+        if ('object' == typeof screenshot && 'base64' in screenshot) return screenshot.base64;
+        if ('string' == typeof screenshot) return screenshot;
+        return '';
+    }, [
+        screenshot
+    ]);
+    const highlightBoxes = highlightOverlays.map((highlight)=>getCenterHighlightBox(highlight));
+    let bottomTipA = null;
+    if (1 === highlightBoxes.length) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightBoxes[0])
+            ]
+        })
+    });
+    else if (highlightBoxes.length > 1) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightBoxes)
+            ]
+        })
+    });
+    return /*#__PURE__*/ jsxs("div", {
+        className: "blackboard",
+        children: [
+            /*#__PURE__*/ jsxs("div", {
+                className: "blackboard-main-content",
+                style: {
+                    width: 'fit-content',
+                    maxWidth: '100%',
+                    position: 'relative'
+                },
+                children: [
+                    screenshotBase64 && /*#__PURE__*/ jsx("img", {
+                        src: screenshotBase64,
+                        alt: "screenshot",
+                        className: "blackboard-screenshot",
+                        draggable: false
+                    }),
+                    /*#__PURE__*/ jsxs("div", {
+                        className: "blackboard-overlay",
+                        style: {
+                            aspectRatio: `${screenWidth}/${screenHeight}`,
+                            '--ui-scale': Math.max(1, Math.sqrt(screenWidth / 1920))
+                        },
+                        children: [
+                            highlightRect && /*#__PURE__*/ jsx("div", {
+                                className: "blackboard-rect blackboard-rect-search",
+                                style: {
+                                    left: `${highlightRect.left / screenWidth * 100}%`,
+                                    top: `${highlightRect.top / screenHeight * 100}%`,
+                                    width: `${highlightRect.width / screenWidth * 100}%`,
+                                    height: `${highlightRect.height / screenHeight * 100}%`
+                                },
+                                children: /*#__PURE__*/ jsx("span", {
+                                    className: "blackboard-rect-label",
+                                    children: "Search Area"
+                                })
+                            }),
+                            highlightOverlays.map((el)=>{
+                                const highlightBox = getCenterHighlightBox(el);
+                                return /*#__PURE__*/ jsx("div", {
+                                    className: "blackboard-rect blackboard-rect-highlight",
+                                    style: {
+                                        left: `${highlightBox.left / screenWidth * 100}%`,
+                                        top: `${highlightBox.top / screenHeight * 100}%`,
+                                        width: `${highlightBox.width / screenWidth * 100}%`,
+                                        height: `${highlightBox.height / screenHeight * 100}%`
+                                    }
+                                }, `${el.key}-rect`);
+                            })
+                        ]
+                    })
+                ]
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "bottom-tip",
+                style: {
+                    display: props.hideController ? 'none' : 'block'
+                },
+                children: bottomTipA
+            })
+        ]
+    });
+};
+const blackboard = Blackboard;
+export { Blackboard, blackboard as default };

package/dist/es/component/config-selector/index.mjs

@@ -0,0 +1,251 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Checkbox, Dropdown, Radio, Tooltip } from "antd";
+import setting from "../../icons/setting.mjs";
+import { useEnvConfig } from "../../store/store.mjs";
+import { alwaysRefreshScreenInfoTip, autoDismissKeyboardTip, deepLocateTip, deepThinkTip, domIncludedTip, imeStrategyTip, keyboardDismissStrategyTip, screenshotIncludedTip, trackingTip } from "../../utils/constants.mjs";
+import { getDeviceCapabilities, hasDeviceSpecificConfig } from "../../utils/device-capabilities.mjs";
+const ConfigSelector = ({ showDeepLocateOption = false, showDeepThinkOption = false, enableTracking = false, showDataExtractionOptions = false, hideDomAndScreenshotOptions = false, deviceType, trigger, popupPlacement = 'bottomRight' })=>{
+    const forceSameTabNavigation = useEnvConfig((state)=>state.forceSameTabNavigation);
+    const setForceSameTabNavigation = useEnvConfig((state)=>state.setForceSameTabNavigation);
+    const deepLocate = useEnvConfig((state)=>state.deepLocate);
+    const setDeepLocate = useEnvConfig((state)=>state.setDeepLocate);
+    const deepThink = useEnvConfig((state)=>state.deepThink);
+    const setDeepThink = useEnvConfig((state)=>state.setDeepThink);
+    const screenshotIncluded = useEnvConfig((state)=>state.screenshotIncluded);
+    const setScreenshotIncluded = useEnvConfig((state)=>state.setScreenshotIncluded);
+    const domIncluded = useEnvConfig((state)=>state.domIncluded);
+    const setDomIncluded = useEnvConfig((state)=>state.setDomIncluded);
+    const imeStrategy = useEnvConfig((state)=>state.imeStrategy);
+    const setImeStrategy = useEnvConfig((state)=>state.setImeStrategy);
+    const autoDismissKeyboard = useEnvConfig((state)=>state.autoDismissKeyboard);
+    const setAutoDismissKeyboard = useEnvConfig((state)=>state.setAutoDismissKeyboard);
+    const keyboardDismissStrategy = useEnvConfig((state)=>state.keyboardDismissStrategy);
+    const setKeyboardDismissStrategy = useEnvConfig((state)=>state.setKeyboardDismissStrategy);
+    const alwaysRefreshScreenInfo = useEnvConfig((state)=>state.alwaysRefreshScreenInfo);
+    const setAlwaysRefreshScreenInfo = useEnvConfig((state)=>state.setAlwaysRefreshScreenInfo);
+    const deviceCapabilities = getDeviceCapabilities(deviceType);
+    const hasDeviceOptions = hasDeviceSpecificConfig(deviceType);
+    if (!enableTracking && !showDeepLocateOption && !showDeepThinkOption && !showDataExtractionOptions && !hasDeviceOptions) return null;
+    const configItems = buildConfigItems();
+    return /*#__PURE__*/ jsx("div", {
+        className: "selector-trigger",
+        children: /*#__PURE__*/ jsx(Dropdown, {
+            getPopupContainer: (triggerNode)=>triggerNode.ownerDocument.body,
+            menu: {
+                items: configItems
+            },
+            overlayClassName: "config-selector-dropdown",
+            overlayStyle: {
+                zIndex: 10010
+            },
+            placement: popupPlacement,
+            trigger: [
+                'click'
+            ],
+            children: null != trigger ? trigger : /*#__PURE__*/ jsx(setting, {
+                width: 24,
+                height: 24
+            })
+        })
+    });
+    function buildConfigItems() {
+        const items = [];
+        if (enableTracking) items.push({
+            label: /*#__PURE__*/ jsx(Checkbox, {
+                onChange: (e)=>setForceSameTabNavigation(e.target.checked),
+                checked: forceSameTabNavigation,
+                children: trackingTip
+            }),
+            key: 'track-config'
+        });
+        if (showDeepLocateOption) items.push({
+            label: /*#__PURE__*/ jsx(Checkbox, {
+                onChange: (e)=>{
+                    setDeepLocate(e.target.checked);
+                },
+                checked: deepLocate,
+                children: deepLocateTip
+            }),
+            key: 'deep-locate-config'
+        });
+        if (showDeepThinkOption) items.push({
+            label: /*#__PURE__*/ jsxs("div", {
+                style: {
+                    padding: '4px 0'
+                },
+                children: [
+                    /*#__PURE__*/ jsx("div", {
+                        style: {
+                            marginBottom: '4px',
+                            fontSize: '14px'
+                        },
+                        children: deepThinkTip
+                    }),
+                    /*#__PURE__*/ jsxs(Radio.Group, {
+                        size: "small",
+                        value: deepThink,
+                        onChange: (e)=>setDeepThink(e.target.value),
+                        children: [
+                            /*#__PURE__*/ jsx(Tooltip, {
+                                title: "Controlled by MIDSCENE_MODEL_REASONING_ENABLED env variable",
+                                children: /*#__PURE__*/ jsx(Radio, {
+                                    value: 'unset',
+                                    children: "Auto"
+                                })
+                            }),
+                            /*#__PURE__*/ jsx(Radio, {
+                                value: true,
+                                children: "On"
+                            }),
+                            /*#__PURE__*/ jsx(Radio, {
+                                value: false,
+                                children: "Off"
+                            })
+                        ]
+                    })
+                ]
+            }),
+            key: 'deep-think-config'
+        });
+        if (showDataExtractionOptions && !hideDomAndScreenshotOptions) {
+            items.push({
+                label: /*#__PURE__*/ jsx(Checkbox, {
+                    onChange: (e)=>{
+                        setScreenshotIncluded(e.target.checked);
+                    },
+                    checked: screenshotIncluded,
+                    children: screenshotIncludedTip
+                }),
+                key: 'screenshot-included-config'
+            });
+            items.push({
+                label: /*#__PURE__*/ jsxs("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ jsx("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: domIncludedTip
+                        }),
+                        /*#__PURE__*/ jsxs(Radio.Group, {
+                            size: "small",
+                            value: domIncluded,
+                            onChange: (e)=>setDomIncluded(e.target.value),
+                            children: [
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: false,
+                                    children: "Off"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: true,
+                                    children: "All"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: 'visible-only',
+                                    children: "Visible only"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'dom-included-config'
+            });
+        }
+        if (deviceCapabilities.supportsImeStrategy) {
+            items.push({
+                label: /*#__PURE__*/ jsxs("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ jsx("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: imeStrategyTip
+                        }),
+                        /*#__PURE__*/ jsxs(Radio.Group, {
+                            size: "small",
+                            value: imeStrategy,
+                            onChange: (e)=>setImeStrategy(e.target.value),
+                            children: [
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: "always-yadb",
+                                    children: "Always YADB"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: "yadb-for-non-ascii",
+                                    children: "YADB for non-ASCII"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'ime-strategy-config'
+            });
+            if (deviceCapabilities.supportsAutoDismissKeyboard) items.push({
+                label: /*#__PURE__*/ jsx(Checkbox, {
+                    onChange: (e)=>setAutoDismissKeyboard(e.target.checked),
+                    checked: autoDismissKeyboard,
+                    children: autoDismissKeyboardTip
+                }),
+                key: 'auto-dismiss-keyboard-config'
+            });
+            if (deviceCapabilities.supportsKeyboardDismissStrategy) items.push({
+                label: /*#__PURE__*/ jsxs("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ jsx("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: keyboardDismissStrategyTip
+                        }),
+                        /*#__PURE__*/ jsxs(Radio.Group, {
+                            size: "small",
+                            value: keyboardDismissStrategy,
+                            onChange: (e)=>setKeyboardDismissStrategy(e.target.value),
+                            children: [
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: "esc-first",
+                                    children: "ESC first"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: "back-first",
+                                    children: "Back first"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'keyboard-dismiss-strategy-config'
+            });
+            if (deviceCapabilities.supportsAlwaysRefreshScreenInfo) items.push({
+                label: /*#__PURE__*/ jsx(Checkbox, {
+                    onChange: (e)=>setAlwaysRefreshScreenInfo(e.target.checked),
+                    checked: alwaysRefreshScreenInfo,
+                    children: alwaysRefreshScreenInfoTip
+                }),
+                key: 'always-refresh-screen-info-config'
+            });
+        }
+        if (!deviceCapabilities.supportsImeStrategy && deviceCapabilities.supportsAutoDismissKeyboard) items.push({
+            label: /*#__PURE__*/ jsx(Checkbox, {
+                onChange: (e)=>setAutoDismissKeyboard(e.target.checked),
+                checked: autoDismissKeyboard,
+                children: autoDismissKeyboardTip
+            }),
+            key: 'auto-dismiss-keyboard-config'
+        });
+        return items;
+    }
+};
+export { ConfigSelector };

package/dist/es/component/context-preview/index.mjs

@@ -0,0 +1,37 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Button } from "antd";
+import blackboard from "../blackboard/index.mjs";
+import { iconForStatus } from "../misc/index.mjs";
+import playground_demo_ui_context from "../playground/playground-demo-ui-context.json";
+const ContextPreview = ({ uiContextPreview, setUiContextPreview, showContextPreview })=>{
+    if (!showContextPreview) return null;
+    return /*#__PURE__*/ jsxs("div", {
+        className: "form-part context-panel",
+        children: [
+            /*#__PURE__*/ jsx("h3", {
+                children: "UI Context"
+            }),
+            uiContextPreview ? /*#__PURE__*/ jsx(blackboard, {
+                uiContext: uiContextPreview,
+                hideController: true
+            }) : /*#__PURE__*/ jsxs("div", {
+                children: [
+                    iconForStatus('failed'),
+                    " No UI context",
+                    /*#__PURE__*/ jsx(Button, {
+                        type: "link",
+                        onClick: (e)=>{
+                            e.preventDefault();
+                            setUiContextPreview(playground_demo_ui_context);
+                        },
+                        children: "Load Demo"
+                    }),
+                    /*#__PURE__*/ jsx("div", {
+                        children: "To load the UI context, you can either use the demo data above, or click the 'Send to Playground' in the report page."
+                    })
+                ]
+            })
+        ]
+    });
+};
+export { ContextPreview };