@sqaitech/visualizer 0.30.10

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,9 @@
+## Documentation
+
+An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language.
+
+See https://SQAIjs.com/ for details.
+
+## License
+
+SQAI is MIT licensed.

package/dist/es/component/blackboard/index.css

@@ -0,0 +1,33 @@
+.blackboard .footer {
+  color: #aaa;
+}
+
+.blackboard ul {
+  padding-left: 0;
+}
+
+.blackboard li {
+  list-style: none;
+}
+
+.blackboard .bottom-tip {
+  height: 30px;
+}
+
+.blackboard .bottom-tip-item {
+  color: #aaa;
+  text-overflow: ellipsis;
+  word-wrap: break-word;
+  max-width: 500px;
+}
+
+.blackboard-filter {
+  margin: 10px 0;
+}
+
+.blackboard-main-content canvas {
+  box-sizing: border-box;
+  border: 1px solid #888;
+  width: 100%;
+}
+

package/dist/es/component/blackboard/index.mjs

@@ -0,0 +1,278 @@
+'use client';
+import { jsx, jsxs } from "react/jsx-runtime";
+import "pixi.js/unsafe-eval";
+import { Checkbox } from "antd";
+import { Application, Container, Graphics, Rectangle, Sprite, Text, Texture } from "pixi.js";
+import { useEffect, useMemo, useRef, useState } from "react";
+import { colorForName, highlightColorForType } from "../../utils/color.mjs";
+import "./index.css";
+import { treeToList } from "@sqaitech/shared/extractor";
+import { DropShadowFilter } from "pixi-filters";
+import { useBlackboardPreference } from "../../store/store.mjs";
+const itemFillAlpha = 0.4;
+const highlightAlpha = 0.4;
+const pointRadius = 10;
+const pointMarkForItem = (point, type)=>{
+    const [x, y] = point;
+    const themeColor = highlightColorForType('element');
+    const graphics = new Graphics();
+    graphics.beginFill(themeColor, itemFillAlpha);
+    graphics.drawCircle(x, y, pointRadius);
+    graphics.endFill();
+    return graphics;
+};
+const rectMarkForItem = (rect, name, type)=>{
+    const { left, top, width, height } = rect;
+    let themeColor;
+    themeColor = 'element' === type ? colorForName(name) : 'searchArea' === type ? highlightColorForType('searchArea') : highlightColorForType('element');
+    const alpha = 'highlight' === type ? highlightAlpha : itemFillAlpha;
+    const graphics = new Graphics();
+    graphics.beginFill(themeColor, alpha);
+    graphics.lineStyle(1, themeColor, 1);
+    graphics.drawRect(left, top, width, height);
+    graphics.endFill();
+    const dropShadowFilter = new DropShadowFilter({
+        blur: 2,
+        quality: 3,
+        alpha: 0.4,
+        offset: {
+            x: 4,
+            y: 4
+        },
+        color: 0x333333
+    });
+    graphics.filters = [
+        dropShadowFilter
+    ];
+    const nameFontSize = 18;
+    if (!name) return [
+        graphics
+    ];
+    const texts = new Text(name, {
+        fontSize: nameFontSize,
+        fill: 0x0
+    });
+    texts.x = left;
+    texts.y = Math.max(top - (nameFontSize + 4), 0);
+    return [
+        graphics,
+        texts
+    ];
+};
+const Blackboard = (props)=>{
+    const highlightElements = props.highlightElements || [];
+    const highlightIds = highlightElements.map((e)=>e.id);
+    const highlightRect = props.highlightRect;
+    const highlightPoints = props.highlightPoints;
+    const context = props.uiContext;
+    const { size, screenshotBase64 } = context;
+    const screenWidth = size.width;
+    const screenHeight = size.height;
+    const domRef = useRef(null);
+    const app = useMemo(()=>new Application(), []);
+    const [appInitialed, setAppInitialed] = useState(false);
+    const highlightContainer = useMemo(()=>new Container(), []);
+    const elementMarkContainer = useMemo(()=>new Container(), []);
+    const [hoverElement, setHoverElement] = useState(null);
+    const pixiBgRef = useRef(void 0);
+    const { backgroundVisible, setBackgroundVisible, elementsVisible, setElementsVisible } = useBlackboardPreference();
+    useEffect(()=>{
+        Promise.resolve((async ()=>{
+            if (!domRef.current || !screenWidth) return;
+            await app.init({
+                width: screenWidth,
+                height: screenHeight,
+                background: 0xffffff
+            });
+            const canvasEl = domRef.current;
+            domRef.current.appendChild(app.canvas);
+            const { clientWidth } = domRef.current.parentElement;
+            const targetHeight = 0.6 * window.innerHeight;
+            const viewportRatio = clientWidth / targetHeight;
+            if (screenWidth / screenHeight <= viewportRatio) {
+                const ratio = targetHeight / screenHeight;
+                canvasEl.style.width = `${Math.floor(screenWidth * ratio)}px`;
+                canvasEl.style.height = `${Math.floor(screenHeight * ratio)}px`;
+            }
+            app.stage.addChild(highlightContainer);
+            app.stage.addChild(elementMarkContainer);
+            setAppInitialed(true);
+        })());
+        return ()=>{
+            console.log('will destroy');
+            try {
+                app.destroy(true, {
+                    children: true,
+                    texture: true
+                });
+            } catch (e) {
+                console.warn('destroy failed', e);
+            }
+        };
+    }, [
+        app,
+        screenWidth,
+        screenHeight
+    ]);
+    useEffect(()=>{
+        if (!appInitialed) return;
+        app.stage.eventMode = 'static';
+        app.stage.hitArea = new Rectangle(0, 0, screenWidth, screenHeight);
+        const clickHandler = (event)=>{
+            var _props_onCanvasClick;
+            console.log('pixi click', event);
+            const { x, y } = event.data.global;
+            null == (_props_onCanvasClick = props.onCanvasClick) || _props_onCanvasClick.call(props, [
+                Math.round(x),
+                Math.round(y)
+            ]);
+        };
+        app.stage.on('click', clickHandler);
+        return ()=>{
+            var _app_stage;
+            null == app || null == (_app_stage = app.stage) || _app_stage.off('click');
+        };
+    }, [
+        appInitialed,
+        props.onCanvasClick,
+        screenWidth,
+        screenHeight
+    ]);
+    useEffect(()=>{
+        if (!appInitialed) return;
+        const img = new Image();
+        img.onload = ()=>{
+            if (!app.stage) return;
+            const screenshotTexture = Texture.from(img);
+            const backgroundSprite = new Sprite(screenshotTexture);
+            backgroundSprite.x = 0;
+            backgroundSprite.y = 0;
+            backgroundSprite.width = screenWidth;
+            backgroundSprite.height = screenHeight;
+            backgroundSprite.eventMode = 'passive';
+            app.stage.addChildAt(backgroundSprite, 0);
+            pixiBgRef.current = backgroundSprite;
+            backgroundSprite.visible = backgroundVisible;
+        };
+        img.onerror = (e)=>{
+            console.error('load screenshot failed', e);
+        };
+        img.src = screenshotBase64;
+    }, [
+        app.stage,
+        appInitialed,
+        screenWidth,
+        screenHeight
+    ]);
+    const { highlightElementRects } = useMemo(()=>{
+        const highlightElementRects = [];
+        highlightContainer.removeChildren();
+        elementMarkContainer.removeChildren();
+        highlightContainer.eventMode = 'passive';
+        elementMarkContainer.eventMode = 'passive';
+        if (highlightRect) {
+            const [graphics] = rectMarkForItem(highlightRect, 'Search Area', 'searchArea');
+            highlightContainer.addChild(graphics);
+        }
+        if (highlightElements.length) highlightElements.forEach((element)=>{
+            const { rect, content, id } = element;
+            const [graphics] = rectMarkForItem(rect, content, 'highlight');
+            highlightContainer.addChild(graphics);
+        });
+        if (null == highlightPoints ? void 0 : highlightPoints.length) highlightPoints.forEach((point)=>{
+            const graphics = pointMarkForItem(point, 'highlightPoint');
+            highlightContainer.addChild(graphics);
+        });
+        const elements = treeToList(context.tree);
+        elements.forEach((element)=>{
+            const { rect, content, id } = element;
+            const ifHighlight = highlightIds.includes(id) || (null == hoverElement ? void 0 : hoverElement.id) === id;
+            if (ifHighlight) return;
+            const [graphics] = rectMarkForItem(rect, content, 'element');
+            elementMarkContainer.addChild(graphics);
+        });
+        elementMarkContainer.visible = elementsVisible;
+        return {
+            highlightElementRects
+        };
+    }, [
+        app,
+        appInitialed,
+        highlightElements,
+        context.tree,
+        hoverElement,
+        highlightRect,
+        highlightPoints
+    ]);
+    const onSetBackgroundVisible = (e)=>{
+        setBackgroundVisible(e.target.checked);
+        if (pixiBgRef.current) pixiBgRef.current.visible = e.target.checked;
+    };
+    const onSetElementsVisible = (e)=>{
+        setElementsVisible(e.target.checked);
+        elementMarkContainer.visible = e.target.checked;
+    };
+    let bottomTipA = null;
+    if (1 === highlightElementRects.length) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightElementRects[0])
+            ]
+        })
+    });
+    else if (highlightElementRects.length > 1) bottomTipA = /*#__PURE__*/ jsx("div", {
+        className: "bottom-tip",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "bottom-tip-item",
+            children: [
+                "Element: ",
+                JSON.stringify(highlightElementRects)
+            ]
+        })
+    });
+    return /*#__PURE__*/ jsxs("div", {
+        className: "blackboard",
+        children: [
+            /*#__PURE__*/ jsx("div", {
+                className: "blackboard-main-content",
+                style: {
+                    width: '100%'
+                },
+                ref: domRef
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "blackboard-filter",
+                style: {
+                    display: props.hideController ? 'none' : 'block'
+                },
+                children: /*#__PURE__*/ jsxs("div", {
+                    className: "overlay-control",
+                    children: [
+                        /*#__PURE__*/ jsx(Checkbox, {
+                            checked: backgroundVisible,
+                            onChange: onSetBackgroundVisible,
+                            children: "Background"
+                        }),
+                        /*#__PURE__*/ jsx(Checkbox, {
+                            checked: elementsVisible,
+                            onChange: onSetElementsVisible,
+                            children: "Elements"
+                        })
+                    ]
+                })
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "bottom-tip",
+                style: {
+                    display: props.hideController ? 'none' : 'block'
+                },
+                children: bottomTipA
+            })
+        ]
+    });
+};
+const blackboard = Blackboard;
+export { Blackboard, blackboard as default, pointMarkForItem, rectMarkForItem };

package/dist/es/component/config-selector/index.mjs

@@ -0,0 +1,104 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Checkbox, Dropdown, Radio } from "antd";
+import setting from "../../icons/setting.mjs";
+import { useEnvConfig } from "../../store/store.mjs";
+import { deepThinkTip, domIncludedTip, screenshotIncludedTip, trackingTip } from "../../utils/constants.mjs";
+const ConfigSelector = (param)=>{
+    let { showDeepThinkOption = false, enableTracking = false, showDataExtractionOptions = false, hideDomAndScreenshotOptions = false } = param;
+    const forceSameTabNavigation = useEnvConfig((state)=>state.forceSameTabNavigation);
+    const setForceSameTabNavigation = useEnvConfig((state)=>state.setForceSameTabNavigation);
+    const deepThink = useEnvConfig((state)=>state.deepThink);
+    const setDeepThink = useEnvConfig((state)=>state.setDeepThink);
+    const screenshotIncluded = useEnvConfig((state)=>state.screenshotIncluded);
+    const setScreenshotIncluded = useEnvConfig((state)=>state.setScreenshotIncluded);
+    const domIncluded = useEnvConfig((state)=>state.domIncluded);
+    const setDomIncluded = useEnvConfig((state)=>state.setDomIncluded);
+    if (!enableTracking && !showDeepThinkOption && !showDataExtractionOptions) return null;
+    const configItems = buildConfigItems();
+    return /*#__PURE__*/ jsx("div", {
+        className: "selector-trigger",
+        children: /*#__PURE__*/ jsx(Dropdown, {
+            menu: {
+                items: configItems
+            },
+            trigger: [
+                'click'
+            ],
+            children: /*#__PURE__*/ jsx(setting, {
+                width: 24,
+                height: 24
+            })
+        })
+    });
+    function buildConfigItems() {
+        const items = [];
+        if (enableTracking) items.push({
+            label: /*#__PURE__*/ jsx(Checkbox, {
+                onChange: (e)=>setForceSameTabNavigation(e.target.checked),
+                checked: forceSameTabNavigation,
+                children: trackingTip
+            }),
+            key: 'track-config'
+        });
+        if (showDeepThinkOption) items.push({
+            label: /*#__PURE__*/ jsx(Checkbox, {
+                onChange: (e)=>{
+                    setDeepThink(e.target.checked);
+                },
+                checked: deepThink,
+                children: deepThinkTip
+            }),
+            key: 'deep-think-config'
+        });
+        if (showDataExtractionOptions && !hideDomAndScreenshotOptions) {
+            items.push({
+                label: /*#__PURE__*/ jsx(Checkbox, {
+                    onChange: (e)=>{
+                        setScreenshotIncluded(e.target.checked);
+                    },
+                    checked: screenshotIncluded,
+                    children: screenshotIncludedTip
+                }),
+                key: 'screenshot-included-config'
+            });
+            items.push({
+                label: /*#__PURE__*/ jsxs("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ jsx("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: domIncludedTip
+                        }),
+                        /*#__PURE__*/ jsxs(Radio.Group, {
+                            size: "small",
+                            value: domIncluded,
+                            onChange: (e)=>setDomIncluded(e.target.value),
+                            children: [
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: false,
+                                    children: "Off"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: true,
+                                    children: "All"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: 'visible-only',
+                                    children: "Visible only"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'dom-included-config'
+            });
+        }
+        return items;
+    }
+};
+export { ConfigSelector };

package/dist/es/component/context-preview/index.mjs

@@ -0,0 +1,38 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Button } from "antd";
+import blackboard from "../blackboard/index.mjs";
+import { iconForStatus } from "../misc/index.mjs";
+import playground_demo_ui_context from "../playground/playground-demo-ui-context.json";
+const ContextPreview = (param)=>{
+    let { uiContextPreview, setUiContextPreview, showContextPreview } = param;
+    if (!showContextPreview) return null;
+    return /*#__PURE__*/ jsxs("div", {
+        className: "form-part context-panel",
+        children: [
+            /*#__PURE__*/ jsx("h3", {
+                children: "UI Context"
+            }),
+            uiContextPreview ? /*#__PURE__*/ jsx(blackboard, {
+                uiContext: uiContextPreview,
+                hideController: true
+            }) : /*#__PURE__*/ jsxs("div", {
+                children: [
+                    iconForStatus('failed'),
+                    " No UI context",
+                    /*#__PURE__*/ jsx(Button, {
+                        type: "link",
+                        onClick: (e)=>{
+                            e.preventDefault();
+                            setUiContextPreview(playground_demo_ui_context);
+                        },
+                        children: "Load Demo"
+                    }),
+                    /*#__PURE__*/ jsx("div", {
+                        children: "To load the UI context, you can either use the demo data above, or click the 'Send to Playground' in the report page."
+                    })
+                ]
+            })
+        ]
+    });
+};
+export { ContextPreview };

package/dist/es/component/env-config/index.mjs

@@ -0,0 +1,112 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { SettingOutlined } from "@ant-design/icons";
+import { Input, Modal, Tooltip } from "antd";
+import { useEffect, useRef, useState } from "react";
+import { useEnvConfig } from "../../store/store.mjs";
+function EnvConfig(param) {
+    let { showTooltipWhenEmpty = true, showModelName = true, tooltipPlacement = 'bottom', mode = 'icon' } = param;
+    const { config, configString, loadConfig, syncFromStorage } = useEnvConfig();
+    const [isModalOpen, setIsModalOpen] = useState(false);
+    const [tempConfigString, setTempConfigString] = useState(configString);
+    const midsceneModelName = config.MIDSCENE_MODEL_NAME;
+    const componentRef = useRef(null);
+    const showModal = (e)=>{
+        syncFromStorage();
+        setIsModalOpen(true);
+        e.preventDefault();
+        e.stopPropagation();
+    };
+    const handleOk = ()=>{
+        setIsModalOpen(false);
+        loadConfig(tempConfigString);
+    };
+    const handleCancel = ()=>{
+        setIsModalOpen(false);
+    };
+    useEffect(()=>{
+        if (isModalOpen) setTempConfigString(configString);
+    }, [
+        isModalOpen,
+        configString
+    ]);
+    return /*#__PURE__*/ jsxs("div", {
+        style: {
+            display: 'flex',
+            justifyContent: 'flex-end',
+            gap: '10px',
+            alignItems: 'center',
+            height: '100%',
+            minHeight: '32px'
+        },
+        ref: componentRef,
+        children: [
+            showModelName ? midsceneModelName : null,
+            /*#__PURE__*/ jsx(Tooltip, {
+                title: "Please set up your environment variables before using.",
+                placement: tooltipPlacement,
+                align: {
+                    offset: [
+                        -10,
+                        5
+                    ]
+                },
+                getPopupContainer: ()=>componentRef.current,
+                open: isModalOpen ? false : showTooltipWhenEmpty ? 0 === Object.keys(config).length : void 0,
+                children: 'icon' === mode ? /*#__PURE__*/ jsx(SettingOutlined, {
+                    onClick: showModal
+                }) : /*#__PURE__*/ jsx("span", {
+                    onClick: showModal,
+                    style: {
+                        color: '#006AFF',
+                        cursor: 'pointer'
+                    },
+                    children: "set up"
+                })
+            }),
+            /*#__PURE__*/ jsxs(Modal, {
+                title: "Model Env Config",
+                open: isModalOpen,
+                onOk: handleOk,
+                onCancel: handleCancel,
+                okText: "Save",
+                style: {
+                    width: '800px',
+                    height: '100%',
+                    marginTop: '10%'
+                },
+                destroyOnClose: true,
+                maskClosable: true,
+                centered: true,
+                children: [
+                    /*#__PURE__*/ jsx(Input.TextArea, {
+                        rows: 7,
+                        placeholder: 'OPENAI_API_KEY=sk-...\nMIDSCENE_MODEL_NAME=gpt-4o-2024-08-06\n...',
+                        value: tempConfigString,
+                        onChange: (e)=>setTempConfigString(e.target.value),
+                        style: {
+                            whiteSpace: 'nowrap',
+                            wordWrap: 'break-word'
+                        }
+                    }),
+                    /*#__PURE__*/ jsxs("div", {
+                        children: [
+                            /*#__PURE__*/ jsx("p", {
+                                children: "The format is KEY=VALUE and separated by new lines."
+                            }),
+                            /*#__PURE__*/ jsxs("p", {
+                                children: [
+                                    "These data will be saved ",
+                                    /*#__PURE__*/ jsx("strong", {
+                                        children: "locally in your browser"
+                                    }),
+                                    "."
+                                ]
+                            })
+                        ]
+                    })
+                ]
+            })
+        ]
+    });
+}
+export { EnvConfig };

package/dist/es/component/env-config-reminder/index.css

@@ -0,0 +1,22 @@
+.env-config-reminder {
+  background: #fff2e8;
+  border-radius: 12px;
+  align-items: center;
+  gap: 12px;
+  margin-bottom: 12px;
+  padding: 12px 16px;
+  display: flex;
+}
+
+.env-config-reminder .reminder-icon {
+  color: #fa541c;
+  width: 16px;
+  height: 16px;
+}
+
+.env-config-reminder .reminder-text {
+  color: #000;
+  flex: 1;
+  font-size: 14px;
+}
+

package/dist/es/component/env-config-reminder/index.mjs

@@ -0,0 +1,28 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { ExclamationCircleFilled } from "@ant-design/icons";
+import { useEnvConfig } from "../../store/store.mjs";
+import { EnvConfig } from "../env-config/index.mjs";
+import "./index.css";
+const EnvConfigReminder = (param)=>{
+    let { className = '' } = param;
+    const { config } = useEnvConfig();
+    const configAlreadySet = Object.keys(config || {}).length >= 1;
+    if (configAlreadySet) return null;
+    return /*#__PURE__*/ jsxs("div", {
+        className: `env-config-reminder ${className}`,
+        children: [
+            /*#__PURE__*/ jsx(ExclamationCircleFilled, {
+                className: "reminder-icon"
+            }),
+            /*#__PURE__*/ jsx("span", {
+                className: "reminder-text",
+                children: "Please set up your environment variables before using."
+            }),
+            /*#__PURE__*/ jsx(EnvConfig, {
+                mode: "text",
+                showTooltipWhenEmpty: false
+            })
+        ]
+    });
+};
+export { EnvConfigReminder };