@midscene/recorder-ui 0.0.0

package/dist/recorder.js

@@ -0,0 +1,323 @@
+import { RECORDER_INPUT_BATCH_DELAY_MS, RECORDER_SCROLL_BATCH_DELAY_MS } from "@midscene/shared/constants";
+import { getElementXpath, isNotContainerElement } from "@midscene/shared/extractor";
+const DEBUG = 'undefined' != typeof localStorage && 'true' === localStorage.getItem('DEBUG');
+function debugLog(...args) {
+    if (DEBUG) console.log('[EventRecorder]', ...args);
+}
+function generateHashId(type, elementRect) {
+    const rectStr = elementRect ? `${elementRect.left}_${elementRect.top}_${elementRect.width}_${elementRect.height}${void 0 !== elementRect.x ? `_${elementRect.x}` : ''}${void 0 !== elementRect.y ? `_${elementRect.y}` : ''}` : 'no_rect';
+    const combined = `${type}_${rectStr}_${Date.now()}`;
+    let hash = 0;
+    for(let i = 0; i < combined.length; i++){
+        const char = combined.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash &= hash;
+    }
+    return Math.abs(hash).toString(36);
+}
+const isSameInputTarget = (event1, event2)=>event1.element === event2.element;
+const isSameScrollTarget = (event1, event2)=>event1.element === event2.element;
+const getLastLabelClick = (events)=>{
+    for(let i = events.length - 1; i >= 0; i--){
+        const event = events[i];
+        if ('click' === event.type && event.isLabelClick) return event;
+    }
+};
+class EventRecorder {
+    isRecording = false;
+    eventCallback;
+    scrollThrottleTimer = null;
+    scrollThrottleDelay = RECORDER_SCROLL_BATCH_DELAY_MS;
+    inputThrottleTimer = null;
+    inputThrottleDelay = RECORDER_INPUT_BATCH_DELAY_MS;
+    lastViewportScroll = null;
+    sessionId;
+    mutationObserver = null;
+    constructor(eventCallback, sessionId){
+        this.eventCallback = eventCallback;
+        this.sessionId = sessionId;
+    }
+    createNavigationEvent(url, title) {
+        return {
+            type: 'navigation',
+            url,
+            title,
+            pageInfo: {
+                width: window.innerWidth,
+                height: window.innerHeight
+            },
+            timestamp: Date.now(),
+            hashId: `navigation_${Date.now()}`
+        };
+    }
+    start() {
+        if (this.isRecording) return void debugLog('Recording already active, ignoring start request');
+        this.isRecording = true;
+        debugLog('Starting event recording');
+        setTimeout(()=>{
+            const navigationEvent = this.createNavigationEvent(window.location.href, document.title);
+            this.eventCallback(navigationEvent);
+            debugLog('Added final navigation event', navigationEvent);
+        }, 0);
+        document.addEventListener('click', this.handleClick, true);
+        document.addEventListener('input', this.handleInput);
+        document.addEventListener('scroll', this.handleScroll, {
+            capture: true,
+            passive: true
+        });
+    }
+    stop() {
+        if (!this.isRecording) return void debugLog('Recording not active, ignoring stop request');
+        this.isRecording = false;
+        debugLog('Stopping event recording');
+        if (this.scrollThrottleTimer) {
+            clearTimeout(this.scrollThrottleTimer);
+            this.scrollThrottleTimer = null;
+        }
+        if (this.inputThrottleTimer) {
+            clearTimeout(this.inputThrottleTimer);
+            this.inputThrottleTimer = null;
+        }
+        document.removeEventListener('click', this.handleClick, true);
+        document.removeEventListener('input', this.handleInput);
+        document.removeEventListener('scroll', this.handleScroll, true);
+        debugLog('Removed all event listeners');
+    }
+    handleClick = (event)=>{
+        if (!this.isRecording) return;
+        const target = event.target;
+        const { isLabelClick, labelInfo } = this.checkLabelClick(target);
+        const rect = target.getBoundingClientRect();
+        const elementRect = {
+            x: Number(event.clientX.toFixed(2)),
+            y: Number(event.clientY.toFixed(2))
+        };
+        console.log('isNotContainerElement', isNotContainerElement(target));
+        if (isNotContainerElement(target)) {
+            elementRect.left = Number(rect.left.toFixed(2));
+            elementRect.top = Number(rect.top.toFixed(2));
+            elementRect.width = Number(rect.width.toFixed(2));
+            elementRect.height = Number(rect.height.toFixed(2));
+        }
+        const clickEvent = {
+            type: 'click',
+            elementRect,
+            pageInfo: {
+                width: window.innerWidth,
+                height: window.innerHeight
+            },
+            value: '',
+            timestamp: Date.now(),
+            hashId: generateHashId('click', {
+                ...elementRect
+            }),
+            element: target,
+            isLabelClick,
+            labelInfo,
+            isTrusted: event.isTrusted,
+            detail: event.detail
+        };
+        this.eventCallback(clickEvent);
+    };
+    handleScroll = (event)=>{
+        if (!this.isRecording) return;
+        function isDocument(target) {
+            return target instanceof Document;
+        }
+        const target = event.target;
+        const scrollXTarget = isDocument(target) ? window.scrollX : target.scrollLeft;
+        const scrollYTarget = isDocument(target) ? window.scrollY : target.scrollTop;
+        const rect = isDocument(target) ? {
+            left: 0,
+            top: 0,
+            width: window.innerWidth,
+            height: window.innerHeight
+        } : target.getBoundingClientRect();
+        if (this.scrollThrottleTimer) clearTimeout(this.scrollThrottleTimer);
+        this.scrollThrottleTimer = window.setTimeout(()=>{
+            if (this.isRecording) {
+                const elementRect = {
+                    left: isDocument(target) ? 0 : Number(rect.left.toFixed(2)),
+                    top: isDocument(target) ? 0 : Number(rect.top.toFixed(2)),
+                    width: isDocument(target) ? window.innerWidth : Number(rect.width.toFixed(2)),
+                    height: isDocument(target) ? window.innerHeight : Number(rect.height.toFixed(2))
+                };
+                const scrollEvent = {
+                    type: 'scroll',
+                    elementRect,
+                    pageInfo: {
+                        width: window.innerWidth,
+                        height: window.innerHeight
+                    },
+                    value: `${scrollXTarget.toFixed(2)},${scrollYTarget.toFixed(2)}`,
+                    timestamp: Date.now(),
+                    hashId: generateHashId('scroll', {
+                        ...elementRect
+                    }),
+                    element: target
+                };
+                this.eventCallback(scrollEvent);
+            }
+            this.scrollThrottleTimer = null;
+        }, this.scrollThrottleDelay);
+    };
+    handleInput = (event)=>{
+        if (!this.isRecording) return;
+        const target = event.target;
+        if ('checkbox' === target.type) return;
+        const rect = target.getBoundingClientRect();
+        const elementRect = {
+            left: Number(rect.left.toFixed(2)),
+            top: Number(rect.top.toFixed(2)),
+            width: Number(rect.width.toFixed(2)),
+            height: Number(rect.height.toFixed(2))
+        };
+        if (this.inputThrottleTimer) clearTimeout(this.inputThrottleTimer);
+        this.inputThrottleTimer = window.setTimeout(()=>{
+            if (this.isRecording) {
+                const inputEvent = {
+                    type: 'input',
+                    value: 'password' !== target.type ? target.value : '*****',
+                    timestamp: Date.now(),
+                    hashId: generateHashId('input', {
+                        ...elementRect
+                    }),
+                    element: target,
+                    inputType: target.type || 'text',
+                    elementRect,
+                    pageInfo: {
+                        width: window.innerWidth,
+                        height: window.innerHeight
+                    }
+                };
+                debugLog('Throttled input event:', {
+                    value: inputEvent.value,
+                    timestamp: inputEvent.timestamp,
+                    target: target.tagName,
+                    inputType: target.type
+                });
+                this.eventCallback(inputEvent);
+            }
+            this.inputThrottleTimer = null;
+        }, this.inputThrottleDelay);
+    };
+    checkLabelClick(target) {
+        let isLabelClick = false;
+        let labelInfo;
+        if (target) if ('LABEL' === target.tagName) {
+            isLabelClick = true;
+            labelInfo = {
+                htmlFor: target.htmlFor,
+                textContent: target.textContent?.trim(),
+                xpath: getElementXpath(target)
+            };
+        } else {
+            let parent = target.parentElement;
+            while(parent){
+                if ('LABEL' === parent.tagName) {
+                    isLabelClick = true;
+                    labelInfo = {
+                        htmlFor: parent.htmlFor,
+                        textContent: parent.textContent?.trim(),
+                        xpath: getElementXpath(parent)
+                    };
+                    break;
+                }
+                parent = parent.parentElement;
+            }
+        }
+        return {
+            isLabelClick,
+            labelInfo
+        };
+    }
+    isActive() {
+        return this.isRecording;
+    }
+    optimizeEvent(event, events) {
+        const lastEvent = events[events.length - 1];
+        if ('click' === event.type) {
+            const lastEvent = getLastLabelClick(events);
+            if (event.element) {
+                const { isLabelClick, labelInfo } = this.checkLabelClick(event.element);
+                if (lastEvent && isLabelClick && 'click' === lastEvent.type && lastEvent.isLabelClick && (lastEvent.labelInfo?.htmlFor && event.element.id && lastEvent.labelInfo?.htmlFor === event.element.id || labelInfo?.xpath && lastEvent.labelInfo?.xpath && lastEvent.labelInfo?.xpath === labelInfo?.xpath)) {
+                    debugLog('Skip input event triggered by label click:', event.element);
+                    return events;
+                }
+                return [
+                    ...events,
+                    event
+                ];
+            }
+        }
+        if ('input' === event.type) {
+            if (lastEvent && 'click' === lastEvent.type && lastEvent.isLabelClick && lastEvent.labelInfo?.htmlFor === event.targetId) {
+                debugLog('Skipping input event - triggered by label click:', {
+                    labelHtmlFor: getLastLabelClick(events)?.labelInfo?.htmlFor,
+                    inputId: event.targetId,
+                    element: event.element
+                });
+                return events;
+            }
+            if (lastEvent && 'input' === lastEvent.type && isSameInputTarget(lastEvent, event)) {
+                const oldInputEvent = events[events.length - 1];
+                const newEvents = [
+                    ...events
+                ];
+                newEvents[events.length - 1] = {
+                    value: event.element?.value,
+                    ...event
+                };
+                debugLog('Merging input event:', {
+                    oldValue: oldInputEvent.value,
+                    newValue: event.value,
+                    oldTimestamp: oldInputEvent.timestamp,
+                    newTimestamp: event.timestamp,
+                    target: event.targetTagName
+                });
+                return newEvents;
+            }
+        }
+        if ('scroll' === event.type) {
+            if (lastEvent && 'scroll' === lastEvent.type && isSameScrollTarget(lastEvent, event)) {
+                const oldScrollEvent = events[events.length - 1];
+                const newEvents = [
+                    ...events
+                ];
+                newEvents[events.length - 1] = event;
+                debugLog('Replacing last scroll event with new scroll event:', {
+                    oldPosition: `${oldScrollEvent.elementRect?.left},${oldScrollEvent.elementRect?.top}`,
+                    newPosition: `${event.elementRect?.left},${event.elementRect?.top}`,
+                    oldTimestamp: oldScrollEvent.timestamp,
+                    newTimestamp: event.timestamp,
+                    target: event.targetTagName
+                });
+                return newEvents;
+            }
+        }
+        return [
+            ...events,
+            event
+        ];
+    }
+}
+function convertToChromeEvent(event) {
+    return {
+        type: event.type,
+        url: event.url,
+        title: event.title,
+        value: event.value,
+        elementRect: event.elementRect,
+        pageInfo: event.pageInfo,
+        screenshotBefore: event.screenshotBefore,
+        screenshotAfter: event.screenshotAfter,
+        semantic: event.semantic,
+        screenshotWithBox: event.screenshotWithBox,
+        timestamp: event.timestamp,
+        hashId: event.hashId
+    };
+}
+function convertToChromeEvents(events) {
+    return events.map(convertToChromeEvent);
+}
+export { EventRecorder, convertToChromeEvent, convertToChromeEvents };

package/dist/types/src/Button.d.ts

@@ -0,0 +1,10 @@
+import './button.css';
+interface ButtonProps {
+    primary?: boolean;
+    backgroundColor?: string;
+    size?: 'small' | 'medium' | 'large';
+    label: string;
+    onClick?: () => void;
+}
+export declare const Button: ({ primary, size, backgroundColor, label, ...props }: ButtonProps) => import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/types/src/RecordTimeline.d.ts

@@ -0,0 +1,9 @@
+import type { RecordedEvent } from './recorder';
+import './RecordTimeline.css';
+interface RecordTimelineProps {
+    events: RecordedEvent[];
+    onEventClick?: (event: RecordedEvent, index: number) => void;
+    variant?: 'default' | 'chrome-extension';
+}
+export declare const RecordTimeline: ({ events, onEventClick, variant, }: RecordTimelineProps) => import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/types/src/components/shiny-text.d.ts

@@ -0,0 +1,12 @@
+import type React from 'react';
+import './shiny-text.css';
+type ColorTheme = 'blue' | 'purple' | 'green' | 'rainbow';
+interface ShinyTextProps {
+    text: string;
+    disabled?: boolean;
+    speed?: number;
+    className?: string;
+    colorTheme?: ColorTheme;
+}
+export declare const ShinyText: React.FC<ShinyTextProps>;
+export {};

package/dist/types/src/index.d.ts

@@ -0,0 +1,3 @@
+export { Button } from './Button';
+export { EventRecorder, type RecordedEvent, type ChromeRecordedEvent, convertToChromeEvent, convertToChromeEvents, } from './recorder';
+export { RecordTimeline } from './RecordTimeline';

package/dist/types/src/recorder-iife-index.d.ts

@@ -0,0 +1,6 @@
+import { EventRecorder } from './recorder';
+declare global {
+    interface Window {
+        EventRecorder: typeof EventRecorder;
+    }
+}

package/dist/types/src/recorder.d.ts

@@ -0,0 +1,41 @@
+import type { MidsceneRecorderEvent } from '@midscene/shared/recorder';
+export type ChromeRecordedEvent = MidsceneRecorderEvent;
+export interface RecordedEvent extends ChromeRecordedEvent {
+    element?: HTMLElement;
+    targetTagName?: string;
+    targetId?: string;
+    targetClassName?: string;
+    isLabelClick?: boolean;
+    labelInfo?: {
+        htmlFor?: string;
+        textContent?: string;
+        xpath?: string;
+    };
+    isTrusted?: boolean;
+    detail?: number;
+    inputType?: string;
+}
+export type EventCallback = (event: RecordedEvent) => void;
+export declare class EventRecorder {
+    private isRecording;
+    private eventCallback;
+    private scrollThrottleTimer;
+    private scrollThrottleDelay;
+    private inputThrottleTimer;
+    private inputThrottleDelay;
+    private lastViewportScroll;
+    private sessionId;
+    private mutationObserver;
+    constructor(eventCallback: EventCallback, sessionId: string);
+    createNavigationEvent(url: string, title: string): ChromeRecordedEvent;
+    start(): void;
+    stop(): void;
+    private handleClick;
+    private handleScroll;
+    private handleInput;
+    private checkLabelClick;
+    isActive(): boolean;
+    optimizeEvent(event: RecordedEvent, events: RecordedEvent[]): RecordedEvent[];
+}
+export declare function convertToChromeEvent(event: RecordedEvent): ChromeRecordedEvent;
+export declare function convertToChromeEvents(events: RecordedEvent[]): ChromeRecordedEvent[];

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@midscene/recorder-ui",
+  "version": "0.0.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/web-infra-dev/midscene.git",
+    "directory": "packages/recorder"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/types/src/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "types": "./dist/types/src/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@rsbuild/plugin-react": "^1.4.1",
+    "@rslib/core": "^0.18.3",
+    "@types/react": "^18.3.1",
+    "react": "18.3.1",
+    "typescript": "^5.8.3"
+  },
+  "dependencies": {
+    "@ant-design/icons": "^5.3.1",
+    "antd": "^5.21.6",
+    "dayjs": "^1.11.11",
+    "react-dom": "18.3.1",
+    "@midscene/shared": "1.10.11"
+  },
+  "peerDependencies": {
+    "react": "18.3.1",
+    "react-dom": "18.3.1"
+  },
+  "scripts": {
+    "build": "rslib build",
+    "dev": "npm run build:watch",
+    "build:watch": "rslib build --watch --no-clean"
+  }
+}