@xbrowser/cli 1.9.8 → 1.10.0

package/dist/human-interaction-5AO42MBA.js

@@ -0,0 +1,8 @@
+import {
+  HumanInteractionManager
+} from "./chunk-MWFHZUIY.js";
+import "./chunk-IIM5GOD7.js";
+import "./chunk-KFQGP6VL.js";
+export {
+  HumanInteractionManager
+};

package/dist/{human-interaction-2DZK4MW7.js → human-interaction-C5OEGXGO.js}

@@ -1,6 +1,6 @@
 import {
   HumanInteractionManager
-} from "./chunk-ZJHQPMCY.js";
+} from "./chunk-DWDXEGVK.js";
 import "./chunk-KFQGP6VL.js";
 export {
   HumanInteractionManager

package/dist/human-interaction-ISXZTAYY.js

@@ -0,0 +1,8 @@
+import {
+  HumanInteractionManager
+} from "./chunk-SQHSZENE.js";
+import "./chunk-BNO7OKO4.js";
+import "./chunk-KFQGP6VL.js";
+export {
+  HumanInteractionManager
+};

package/dist/index.d.ts

@@ -348,7 +348,7 @@ interface XBFillOptions {
     noWaitAfter?: boolean;
 }
 interface XBScreenshotOptions {
-    type?: 'png' | 'jpeg';
+    type?: 'png' | 'jpeg' | 'webp';
     quality?: number;
     fullPage?: boolean;
     clip?: {
@@ -490,6 +490,16 @@ type WSMessage = {
 } | {
     type: 'views_update';
     views: ViewInfo[];
+} | {
+    type: 'health_pong';
+    ts: number;
+} | {
+    type: 'snapshot_result';
+    data: {
+        data: string;
+        format: string;
+    } | null;
+    error?: string;
 } | {
     type: 'error';
     data: {
@@ -528,6 +538,15 @@ type WSInboundMessage = {
     deltaY: number;
 } | {
     type: 'solved';
+} | {
+    type: 'reconnect';
+} | {
+    type: 'health_ping';
+    ts: number;
+} | {
+    type: 'snapshot_request';
+    format?: string;
+    quality?: number;
 } | {
     type: 'bind';
     sessionId: string;
@@ -649,6 +668,8 @@ declare class WSServer extends EventEmitter {
         width?: number;
         height?: number;
     }): void;
+    /** Restart screencast for a session — used by viewer reconnect button. */
+    reconnectSession(sessionId: string): Promise<void>;
     unregisterSession(sessionId: string): void;
     pauseScreencast(sessionId: string): Promise<void>;
     resumeScreencast(sessionId: string): Promise<void>;
@@ -738,6 +759,7 @@ declare function executeCommand(commandName: string, params: Record<string, unkn
     cdpEndpoint?: string;
     skipCleanup?: boolean;
     timeout?: number;
+    _recoveryAttempted?: boolean;
 }): Promise<ExecutionResult>;
 /**
  * Execute a chain of browser commands parsed from a string expression.
@@ -1648,6 +1670,8 @@ interface ClickContextItem {
     tag?: string;
     disabled?: boolean;
     href?: string;
+    /** Selector of this item, used for replay and AI-driven clicks. */
+    selector?: string;
 }
 interface ClickContextElement {
     tag: string;
@@ -1677,9 +1701,42 @@ interface ClickContext {
     disappeared: unknown[];
     stateChanges: ClickContextStateChange[];
 }
+/**
+ * Popup/tooltip/dropdown that appears after a hover action.
+ *
+ * Captured via async sampling (200/500/1000ms after `mouseover`) plus a
+ * short-lived MutationObserver. Each entry records the container that popped
+ * up and the visible menu items inside it, so that replay and AI inspection
+ * can correlate the hover trigger with the options that became available.
+ */
+interface HoverPopupInfo {
+    tag: string;
+    selector: string;
+    role?: string;
+    text: string;
+    rect: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    /** Visible menu items (up to 20) — text + selector for replay targeting. */
+    items: ClickContextItem[];
+}
+interface HoverContext {
+    /** Popups that appeared after the hover trigger. */
+    appeared: HoverPopupInfo[];
+    /** Popups that disappeared while the hover was active. */
+    disappeared: Array<{
+        selector: string;
+        reason: string;
+    }>;
+    /** Trigger element's aria-expanded / aria-haspopup state changes. */
+    stateChanges: ClickContextStateChange[];
+}
 interface UserAction {
     id: number;
-    type: 'click' | 'input' | 'change' | 'keydown' | 'submit' | 'scroll' | 'navigation' | 'goto' | 'cdp-fill' | 'cdp-click' | 'cdp-eval' | 'filechooser' | 'dblclick' | 'contextmenu' | 'hover' | 'drag' | 'resize' | 'clipboard' | 'touch' | 'focus' | 'visibility';
+    type: 'click' | 'input' | 'change' | 'keydown' | 'submit' | 'scroll' | 'navigation' | 'goto' | 'cdp-fill' | 'cdp-click' | 'cdp-eval' | 'filechooser' | 'dblclick' | 'contextmenu' | 'hover' | 'drag' | 'resize' | 'clipboard' | 'touch' | 'focus' | 'visibility' | 'popup_appear' | 'discovered_filters';
     timestamp: number;
     url: string;
     pageTitle: string;
@@ -1711,6 +1768,8 @@ interface UserAction {
     scrollX?: number;
     scrollY?: number;
     clickContext?: ClickContext;
+    /** Popups/tooltip/dropdown observed after a `hover` action (analog of clickContext). */
+    hoverContext?: HoverContext;
     files?: {
         names: string[];
         count: number;
@@ -1770,6 +1829,50 @@ interface UserAction {
     };
     /** Base64-encoded PNG screenshot of the target element (captured on key actions) */
     elementScreenshot?: string;
+    /**
+     * Auto-captured snapshot for this action (key types only).
+     * PNG stored in .tmp/snapshots/ (easy to clean up);
+     * aria tree stored inline as compact text (searchable via grep).
+     */
+    snapshots?: {
+        /** Viewport screenshot path (relative, under .tmp/snapshots/) */
+        png?: string;
+        /** Compact accessibility tree text (inline, grep-able) */
+        aria?: string;
+    };
+    /** For type='popup_appear' — proactive sensing of a popover/dropdown/menu
+     * becoming visible (whether triggered by user hover/click, or auto-shown). */
+    popupAppear?: {
+        /** The trigger element that caused the popup (if known) */
+        trigger?: {
+            selector: string;
+            text: string;
+        };
+        /** The popup container */
+        popup: {
+            selector: string;
+            text: string;
+            rect: {
+                x: number;
+                y: number;
+                w: number;
+                h: number;
+            };
+            /** Menu items inside the popup */
+            items: Array<{
+                text: string;
+                selector: string;
+                disabled?: boolean;
+            }>;
+        };
+        /** What caused the popup to appear */
+        cause: 'user-hover' | 'user-click' | 'auto' | 'script';
+        /** True if a real user action triggered it (false for auto-shown popups) */
+        userTriggered: boolean;
+    };
+    /** For type='discovered_filters' — proactive baseline scan of the page.
+     * Pushed on start and after navigation; merged into RecordingData.discoveredFilters. */
+    discoveredFilters?: DiscoveredFilter[];
 }
 interface NetworkEntry {
     id: number;
@@ -1843,6 +1946,32 @@ interface CheckpointEntry {
     relatedActionId?: number;
     context?: Record<string, unknown>;
 }
+interface DiscoveredTrigger {
+    /** Selector for replay/AI targeting */
+    selector: string;
+    /** Text label (e.g. "新发布", "综合", "价格") */
+    text: string;
+    /** Semantic role, if any */
+    role?: string;
+    /** Category — helps AI understand what this trigger does */
+    category: 'sort' | 'filter' | 'tab' | 'menu' | 'navigation' | 'unknown';
+    /** Set to true once we observe a popup appearing for this trigger */
+    hasPopup: boolean;
+    /** Set to true when the user actually clicks or hovers it */
+    userInteracted: boolean;
+    /** Set to true when the mouse lingered on it (>800ms) without clicking */
+    explored: boolean;
+}
+interface DiscoveredFilter {
+    /** Container element selector (the surrounding bar/region) */
+    containerSelector: string;
+    /** Container category */
+    category: 'sort' | 'filter' | 'tab' | 'menu' | 'navigation';
+    /** Container text (first 60 chars, for AI context) */
+    containerText: string;
+    /** Triggers found inside this container */
+    triggers: DiscoveredTrigger[];
+}
 interface RecordingData {
     startUrl: string;
     sessionName: string;
@@ -1851,6 +1980,9 @@ interface RecordingData {
     network: NetworkEntry[];
     contextChanges: ContextChange[];
     checkpoints: CheckpointEntry[];
+    /** Proactive baseline scan of filter/sort/tab/menu regions on the page.
+     * Populated on start and after navigation; updated as popups are observed. */
+    discoveredFilters?: DiscoveredFilter[];
 }
 interface RecordingControlFile {
     pid: number;
@@ -1869,6 +2001,9 @@ declare class SessionRecorder {
     private network;
     private contextChanges;
     private checkpoints;
+    /** Proactive baseline scan of filter/sort/tab/menu regions. Keyed by
+     * containerSelector so re-scans after navigation replace rather than dup. */
+    private discoveredFilters;
     private actionCounter;
     private networkCounter;
     private contextCounter;
@@ -1887,6 +2022,9 @@ declare class SessionRecorder {
     private _lastRequestTs;
     private _isRecording;
     private streamMode;
+    /** Snapshot throttle: prevents too many screenshots during rapid inputs.
+     *  Keyed by 'type|elementSelector', value = last snapshot timestamp. */
+    private snapshotThrottle;
     /** Ambient action types: filtered out in clean mode */
     private static readonly AMBIENT_TYPES;
     constructor(context: XBContext, page: XBPage, sessionName: string);
@@ -1907,6 +2045,8 @@ declare class SessionRecorder {
     addManualCheckpoint(type: string, hint: string, selector?: string): CheckpointEntry;
     /** Directory for this session's recordings. */
     get recordingsDir(): string;
+    /** Directory for auto-captured snapshots (PNG screenshots, stored in .tmp for easy cleanup). */
+    get snapshotsDir(): string;
     static getRecordingsDir(sessionName: string): string;
     /** Path to the control file (used by record stop to signal this process). */
     get controlFilePath(): string;
@@ -1924,6 +2064,12 @@ declare class SessionRecorder {
     static sendStopSignal(sessionName: string): Promise<RecordingControlFile | null>;
     static readSummary(sessionName: string): RecordingSummary | null;
     static readData(sessionName: string): RecordingData | null;
+    /** Whether the initial action-signal script injection succeeded. Check this
+     * after start() returns — if false, user actions will NOT be recorded and
+     * the caller should surface a clear error to the user instead of silently
+     * pretending recording is working. */
+    private _injectionFailed;
+    get injectionFailed(): boolean;
     private injectActionScript;
     private handleRequest;
     private handleResponse;
@@ -1946,6 +2092,25 @@ declare class SessionRecorder {
      *   3. fallback to viewport screenshot (so we always get *something*)
      */
     private captureElementScreenshot;
+    /**
+     * Action types that trigger auto-snapshot capture.
+     * Hover is excluded — it's too noisy and rarely changes the page visually.
+     * Popup_appear captures the popup state (after only).
+     */
+    private static readonly SNAPSHOT_TYPES;
+    /**
+     * Throttle key for snapshot capture to avoid too many during rapid inputs.
+     * Returns true if this action should be throttled (skip snapshot).
+     */
+    private isSnapshotThrottled;
+    /**
+     * Capture a viewport screenshot + compact accessibility tree for a key action.
+     * - PNG stored in .tmp/snapshots/ (temporary, easy to clean up)
+     * - aria tree stored inline as compact text on the action (searchable via grep)
+     *
+     * Non-blocking (errors silently caught). Throttled for rapid inputs.
+     */
+    private captureAutoSnapshot;
     /**
      * After a click, wait 300ms then scan for popover/dropdown/menu elements
      * near the click position. This runs server-side to avoid race conditions