@limrun/ui 0.9.0-rc.4 → 0.9.0-rc.6

package/dist/components/inspect-overlay.d.ts

@@ -0,0 +1,33 @@
+import { default as React } from 'react';
+import { AxElement, AxSnapshot } from '../core/ax-tree';
+export interface InspectOverlayGeometry {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+}
+export type InspectMode = 'select' | 'hover-only';
+export interface InspectOverlayProps {
+    snapshot: AxSnapshot | null;
+    geometry: InspectOverlayGeometry | null;
+    highlightedId: string | null;
+    selectedId: string | null;
+    mode: InspectMode;
+    cursorPosition: {
+        x: number;
+        y: number;
+    } | null;
+    frozenCursorPosition: {
+        x: number;
+        y: number;
+    } | null;
+    onSelectChange: (element: AxElement | null, clickPosition?: {
+        x: number;
+        y: number;
+    } | null) => void;
+    onTapElement: (element: AxElement, tapAt: {
+        x: number;
+        y: number;
+    }) => void;
+}
+export declare const InspectOverlay: React.NamedExoticComponent<InspectOverlayProps>;

package/dist/components/remote-control.d.ts

@@ -1,4 +1,6 @@
 import { default as React } from 'react';
+import { AxStatus } from '../core/ax-fetcher';
+import { AxElement, AxSnapshot } from '../core/ax-tree';
 declare global {
     interface Window {
         debugRemoteControl?: boolean;
@@ -12,6 +14,85 @@ interface RemoteControlProps {
     openUrl?: string;
     showFrame?: boolean;
     autoReconnect?: boolean;
+    /**
+     * Enable the inspect overlay. When set, the component starts polling the
+     * accessibility tree and draws boxes over each element on top of the
+     * video stream.
+     *
+     * - `true` — Select mode. Boxes are clickable, click pins a selection
+     *   with action buttons (Tap / Copy selector / Copy id), ESC clears.
+     *   Device input is blocked while in this mode.
+     * - `'hover-only'` — Boxes follow the cursor as a visual preview. Device
+     *   input still passes through, so you can drive the simulator while
+     *   inspecting.
+     * - `undefined` / `false` (default) — overlay disabled, no polling.
+     */
+    inspectMode?: boolean | 'hover-only';
+    /**
+     * Fires whenever a fresh accessibility snapshot is delivered.
+     *
+     * Customers use this to drive their own side panels, agent prompts,
+     * analytics, etc. The built-in overlay does not require this callback —
+     * it renders from internal state regardless.
+     *
+     * Identical-to-previous snapshots (per `axSnapshotsEqual`) are NOT
+     * re-emitted, so a stable UI doesn't generate callback noise.
+     *
+     * Invoked in a microtask so customer code doesn't run synchronously
+     * inside React's commit phase.
+     */
+    onAxSnapshotChange?: (snapshot: AxSnapshot | null) => void;
+    /**
+     * Fires when the user clicks an overlay element (only emitted when
+     * `inspectMode === true`). `null` indicates a deselection (ESC, click
+     * outside any box, or programmatic clear).
+     *
+     * The `snapshot` field is the snapshot active at the moment of the
+     * click — useful for capturing context without races against the next
+     * poll cycle.
+     */
+    onInspectSelectionChange?: (selection: {
+        element: AxElement;
+        snapshot: AxSnapshot;
+    } | null) => void;
+    /**
+     * Fires whenever the accessibility subsystem changes coarse-grained
+     * status. Useful for rendering readiness indicators or error banners in
+     * a customer-built side panel.
+     *
+     * Transitions are deduplicated; no self-loops are emitted. The `error`
+     * argument is populated when status is `error` or `unavailable`.
+     *
+     * Lifecycle: `idle` → `starting` → `ready` (or `unavailable` / `error`).
+     * Recovery from `error` / `unavailable` is automatic — the fetcher
+     * keeps polling and transitions back to `ready` on the next success.
+     */
+    onAxStatusChange?: (status: AxStatus, error?: string) => void;
+    /**
+     * Base interval (ms) between successful AX-tree fetches.
+     *
+     * The fetcher will:
+     * - Wait `axPollIntervalMs` after a successful fetch with NEW data.
+     * - Double the wait (up to `axMaxBackoffMs`) when consecutive snapshots
+     *   are byte-identical (e.g. static screen).
+     * - Wait 5 s when the server reports AX is unavailable.
+     *
+     * In addition, after user input (taps, scrolls, keypresses, openUrl,
+     * terminateApp, orientation flips), the fetcher enters a short
+     * "activity boost" window (~1.2 s) during which fetches happen at
+     * ~250 ms regardless of this setting. This captures mid-animation UI
+     * changes without you having to manually call `refreshAxTree`.
+     *
+     * @default 500
+     */
+    axPollIntervalMs?: number;
+    /**
+     * Maximum backoff (ms) for the AX-tree polling loop when consecutive
+     * snapshots are unchanged.
+     *
+     * @default 2000
+     */
+    axMaxBackoffMs?: number;
 }
 interface ScreenshotData {
     dataUri: string;
@@ -30,6 +111,11 @@ export interface RemoteControlHandle {
     screenshot: () => Promise<ScreenshotData>;
     terminateApp: (bundleId: string) => Promise<void>;
     reconnect: () => void;
+    refreshAxTree: () => Promise<AxSnapshot>;
+    getAxSnapshot: () => AxSnapshot | null;
+    setInspectHighlight: (element: AxElement | null) => void;
+    setInspectSelection: (element: AxElement | null) => void;
+    getAxStatus: () => AxStatus;
 }
 export declare const RemoteControl: React.ForwardRefExoticComponent<RemoteControlProps & React.RefAttributes<RemoteControlHandle>>;
 export {};

package/dist/core/ax-fetcher.d.ts

@@ -0,0 +1,49 @@
+import { AxPlatform, AxSnapshot } from './ax-tree';
+export type AxStatus = 'idle' | 'starting' | 'ready' | 'unavailable' | 'error';
+export type AxFetcherSendFn = (payload: Record<string, unknown>) => boolean;
+export interface AxFetcherOptions {
+    platform: AxPlatform;
+    send: AxFetcherSendFn;
+    onSnapshot: (snapshot: AxSnapshot | null) => void;
+    onStatusChange?: (status: AxStatus, error?: string) => void;
+    baseIntervalMs?: number;
+    maxBackoffMs?: number;
+}
+export declare class AxFetcher {
+    private readonly platform;
+    private readonly send;
+    private readonly onSnapshot;
+    private readonly onStatusChange?;
+    private readonly baseIntervalMs;
+    private readonly maxBackoffMs;
+    private readonly pending;
+    private running;
+    private timer;
+    private currentInterval;
+    private lastSnapshot;
+    private inflight;
+    private lastBumpAtMs;
+    private boostUntilMs;
+    private status;
+    constructor(opts: AxFetcherOptions);
+    start(): void;
+    stop(): void;
+    getStatus(): AxStatus;
+    private setStatus;
+    bumpActivity(): void;
+    refresh(): Promise<AxSnapshot>;
+    getLatest(): AxSnapshot | null;
+    handleMessage(message: {
+        type?: string;
+        id?: string;
+        [k: string]: unknown;
+    }): boolean;
+    private buildRequest;
+    private requestOnce;
+    private runOnce;
+    private deliver;
+    private scheduleNext;
+    private settleResolve;
+    private settleReject;
+    private failAllPending;
+}

package/dist/core/ax-tree.d.ts

@@ -0,0 +1,99 @@
+export interface AxRect {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface AxSelectors {
+    AXUniqueId?: string;
+    AXLabel?: string;
+    resourceId?: string;
+    contentDesc?: string;
+    text?: string;
+    className?: string;
+}
+export interface AxElement {
+    id: string;
+    path: string;
+    label: string;
+    value: string;
+    role: string;
+    type: string;
+    enabled: boolean;
+    focused: boolean;
+    frame: AxRect;
+    selectors: AxSelectors;
+    raw: Record<string, unknown>;
+}
+export type AxPlatform = 'ios' | 'android';
+export interface AxSnapshot {
+    platform: AxPlatform;
+    screen: {
+        width: number;
+        height: number;
+    };
+    elements: AxElement[];
+    capturedAt: number;
+    errors?: string[];
+}
+export declare const AX_UNAVAILABLE_ERROR = "Accessibility unavailable on this device.";
+interface RawIosNode {
+    AXLabel?: string | null;
+    AXValue?: string | null;
+    AXUniqueId?: string | null;
+    frame?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    role?: string;
+    role_description?: string;
+    type?: string;
+    subrole?: string | null;
+    title?: string | null;
+    enabled?: boolean;
+    focused?: boolean;
+    pid?: number;
+    traits?: string[];
+    children?: RawIosNode[];
+    [key: string]: unknown;
+}
+export declare function normalizeIosTree(roots: RawIosNode[] | RawIosNode): AxSnapshot;
+interface RawAndroidParsedBounds {
+    left: number;
+    top: number;
+    right: number;
+    bottom: number;
+    centerX: number;
+    centerY: number;
+}
+interface RawAndroidNode {
+    index?: string;
+    text?: string;
+    resourceId?: string;
+    className?: string;
+    packageName?: string;
+    contentDesc?: string;
+    clickable?: boolean;
+    enabled?: boolean;
+    focusable?: boolean;
+    focused?: boolean;
+    scrollable?: boolean;
+    selected?: boolean;
+    bounds?: string;
+    parsedBounds?: RawAndroidParsedBounds;
+    [key: string]: unknown;
+}
+export declare function normalizeAndroidTree(nodes: RawAndroidNode[]): AxSnapshot;
+export declare function clampAxFrameForScreen(frame: AxRect, screen: {
+    width: number;
+    height: number;
+}): AxRect | null;
+export declare function axElementsEqual(a: AxElement, b: AxElement): boolean;
+export declare function axSnapshotsEqual(a: AxSnapshot | null, b: AxSnapshot | null): boolean;
+export declare function axElementAtPoint(snapshot: AxSnapshot, x: number, y: number): AxElement | null;
+export declare function axElementSelectorExpression(el: AxElement, platform: AxPlatform): string | null;
+export declare function axElementRoleLabel(el: AxElement): string;
+export declare function axElementSummary(el: AxElement): string;
+export {};