@xpert-ai/chatkit-host-automation 0.3.4 → 0.3.6

package/dist/index.d.ts

@@ -2,7 +2,537 @@ import { ClientToolMessageInput } from '../../chatkit/src/index.ts';
 
 export declare function createHostPageAutomationClientToolHandler(options?: HostPageAutomationOptions): HostPageAutomationClientToolHandler;
 
-export declare const HOST_PAGE_AUTOMATION_TOOL_NAMES: readonly ["host_page_snapshot", "host_page_click", "host_page_fill", "host_page_press", "host_page_select", "host_page_scroll", "host_page_navigate", "host_page_hover", "host_page_focus", "host_page_pointer", "host_page_screenshot", "host_page_wait_for"];
+export declare const HOST_PAGE_AUTOMATION_TOOL_NAMES: readonly ["host_page_snapshot", "host_page_click", "host_page_fill", "host_page_press", "host_page_select", "host_page_scroll", "host_page_navigate", "host_page_hover", "host_page_focus", "host_page_pointer", "host_page_screenshot", "host_page_read", "host_page_wait_for"];
+
+export declare const HOST_PAGE_AUTOMATION_TOOL_SCHEMAS: {
+    readonly host_page_snapshot: {
+        readonly type: "object";
+        readonly properties: {};
+        readonly additionalProperties: false;
+    };
+    readonly host_page_click: {
+        readonly type: "object";
+        readonly properties: {
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_fill: {
+        readonly type: "object";
+        readonly properties: {
+            readonly value: {
+                readonly type: "string";
+                readonly description: "Text value to set on the target field.";
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly required: readonly ["value"];
+        readonly additionalProperties: false;
+    };
+    readonly host_page_press: {
+        readonly type: "object";
+        readonly properties: {
+            readonly key: {
+                readonly type: "string";
+                readonly description: "Keyboard key to press.";
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly required: readonly ["key"];
+        readonly additionalProperties: false;
+    };
+    readonly host_page_select: {
+        readonly type: "object";
+        readonly properties: {
+            readonly value: {
+                readonly type: ["string", "array"];
+                readonly description: "Select option value or values.";
+                readonly items: {
+                    readonly type: "string";
+                };
+            };
+            readonly values: {
+                readonly type: ["string", "array"];
+                readonly description: "Select option values.";
+                readonly items: {
+                    readonly type: "string";
+                };
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_scroll: {
+        readonly type: "object";
+        readonly properties: {
+            readonly deltaX: {
+                readonly type: "number";
+                readonly description: "Horizontal scroll delta.";
+            };
+            readonly deltaY: {
+                readonly type: "number";
+                readonly description: "Vertical scroll delta.";
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_navigate: {
+        readonly type: "object";
+        readonly properties: {
+            readonly url: {
+                readonly type: "string";
+                readonly description: "HTTP(S) URL to navigate to.";
+            };
+        };
+        readonly required: readonly ["url"];
+        readonly additionalProperties: false;
+    };
+    readonly host_page_hover: {
+        readonly type: "object";
+        readonly properties: {
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_focus: {
+        readonly type: "object";
+        readonly properties: {
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_pointer: {
+        readonly type: "object";
+        readonly properties: {
+            readonly action: {
+                readonly type: "string";
+                readonly enum: readonly ["move", "down", "up", "click"];
+                readonly description: "Pointer action. Defaults to click.";
+            };
+            readonly button: {
+                readonly type: ["string", "number"];
+                readonly description: "Mouse button.";
+            };
+            readonly clickCount: {
+                readonly type: "number";
+                readonly description: "Number of clicks for CDP mouse click.";
+            };
+            readonly targetText: {
+                readonly type: "string";
+                readonly description: "Required for explicit coordinate clicks. The actual hit target or near ancestor must contain this text.";
+            };
+            readonly expectedAfterClick: {
+                type: string;
+                properties: {
+                    type: {
+                        const: string;
+                        description: string;
+                    };
+                    field: {
+                        type: string;
+                        description: string;
+                    };
+                    value: {
+                        type: string;
+                        description: string;
+                    };
+                };
+                required: string[];
+                additionalProperties: false;
+            };
+            readonly coordinateSpace: {
+                type: string;
+                enum: string[];
+                description: string;
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_screenshot: {
+        readonly type: "object";
+        readonly properties: {
+            readonly format: {
+                readonly type: "string";
+                readonly enum: readonly ["jpeg", "png"];
+                readonly description: "Screenshot image format.";
+            };
+            readonly quality: {
+                readonly type: "number";
+                readonly description: "JPEG quality from 1 to 100.";
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_read: {
+        readonly type: "object";
+        readonly properties: {
+            readonly blockId: {
+                readonly type: "string";
+                readonly description: "Readable content block id from host_page_snapshot.";
+            };
+            readonly scope: {
+                readonly type: "string";
+                readonly enum: readonly ["visible"];
+                readonly description: "Visible content scope. Use blockId when a specific readable block is known.";
+            };
+            readonly query: {
+                readonly type: "string";
+                readonly description: "Optional text query for selecting relevant readable blocks.";
+            };
+            readonly page: {
+                readonly type: "number";
+                readonly minimum: 1;
+                readonly description: "1-based page for long block content.";
+            };
+            readonly pageSize: {
+                readonly type: "number";
+                readonly minimum: 1;
+                readonly maximum: 100;
+                readonly description: "Items, fields, rows, or blocks to return per page.";
+            };
+            readonly maxChars: {
+                readonly type: "number";
+                readonly minimum: 500;
+                readonly maximum: 12000;
+                readonly description: "Maximum characters to return.";
+            };
+        };
+        readonly additionalProperties: false;
+    };
+    readonly host_page_wait_for: {
+        readonly type: "object";
+        readonly properties: {
+            readonly state: {
+                readonly type: "string";
+                readonly enum: readonly ["attached", "visible", "hidden", "detached"];
+                readonly description: "Target state to wait for. Defaults to visible.";
+            };
+            readonly timeoutSeconds: {
+                readonly type: "number";
+                readonly description: "Maximum wait time in seconds.";
+            };
+            readonly ref: {
+                type: string[];
+                description: string;
+            };
+            readonly axRef: {
+                type: string[];
+                description: string;
+            };
+            readonly selector: {
+                type: string;
+                description: string;
+            };
+            readonly testId: {
+                type: string;
+                description: string;
+            };
+            readonly role: {
+                type: string;
+                description: string;
+            };
+            readonly name: {
+                type: string;
+                description: string;
+            };
+            readonly text: {
+                type: string;
+                description: string;
+            };
+            readonly x: {
+                type: string;
+                description: string;
+            };
+            readonly y: {
+                type: string;
+                description: string;
+            };
+        };
+        readonly additionalProperties: false;
+    };
+};
 
 declare type HostPageAutomationClickEffectContext = {
     point: {
@@ -47,6 +577,17 @@ export declare type HostPageAutomationElementSnapshot = {
     checked?: boolean;
     visible?: boolean;
     actionable?: boolean;
+    receivesEvents?: boolean;
+    occludedBy?: {
+        tag: string;
+        role?: string;
+        name?: string;
+        selector?: string;
+    };
+    safeClickPoints?: Array<{
+        x: number;
+        y: number;
+    }>;
     rect: {
         x: number;
         y: number;
@@ -75,10 +616,12 @@ export declare class HostPageAutomationExecutor {
     private readonly options;
     private refs;
     private nextRef;
+    private readableContent?;
     constructor(options?: HostPageAutomationOptions);
     execute(name: HostPageAutomationToolName, params?: Record<string, unknown>): Promise<unknown>;
     snapshot(): HostPageSnapshot;
     private getRoot;
+    private invalidateReadableContent;
     private getNavigationState;
     private getFrameState;
     private snapshotElement;
@@ -86,6 +629,7 @@ export declare class HostPageAutomationExecutor {
     private createElementSnapshot;
     private resolveElement;
     private click;
+    private read;
     private fill;
     private press;
     private select;
@@ -135,6 +679,79 @@ declare type HostPageAutomationVisualEffectContext = {
 
 declare type HostPageAutomationVisualEffectType = 'click' | 'fill' | 'select' | 'press' | 'scroll' | 'hover' | 'focus' | 'pointer' | 'wait_for' | 'screenshot';
 
+export declare type HostPageReadableContent = {
+    blocks: HostPageReadableContentBlock[];
+    outline?: HostPageReadableContentOutlineItem[];
+    suggestedReads?: HostPageReadableContentSuggestedRead[];
+    totalBlocks: number;
+    truncated: boolean;
+    coverage: {
+        status: 'complete' | 'partial';
+        visibleTextCaptured: boolean;
+        truncatedBlocks: number;
+        collapsedSections: number;
+        crossOriginFrames: number;
+        virtualizedListsDetected: number;
+        visualOnlyRegions: number;
+    };
+    warnings?: string[];
+};
+
+export declare type HostPageReadableContentBlock = {
+    blockId: string;
+    type: HostPageReadableContentBlockType;
+    heading?: string;
+    level?: number;
+    text?: string;
+    items?: string[];
+    fields?: Array<{
+        name: string;
+        value: string;
+    }>;
+    headers?: string[];
+    rows?: string[][];
+    preview: string[];
+    itemCount: number;
+    chars: number;
+    truncated: boolean;
+    rect?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    readHint: {
+        tool: 'host_page_read';
+        args: {
+            blockId: string;
+        };
+    };
+};
+
+declare type HostPageReadableContentBlockType = 'heading' | 'paragraph' | 'list' | 'keyValueList' | 'table';
+
+declare type HostPageReadableContentOutlineItem = {
+    index: number;
+    blockId: string;
+    type: HostPageReadableContentBlockType;
+    heading?: string;
+    level?: number;
+    itemCount: number;
+    chars: number;
+    truncated: boolean;
+};
+
+declare type HostPageReadableContentSuggestedRead = {
+    blockId: string;
+    type: HostPageReadableContentBlockType;
+    heading?: string;
+    reason: string;
+    args: {
+        blockId: string;
+        pageSize?: number;
+    };
+};
+
 export declare type HostPageSnapshot = {
     url: string;
     title: string;
@@ -187,9 +804,23 @@ export declare type HostPageSnapshot = {
         expanded?: boolean;
         focused?: boolean;
     }>;
+    readableContent?: HostPageReadableContent;
     elements: HostPageAutomationElementSnapshot[];
 };
 
 export declare function isHostPageAutomationToolName(value: string): value is HostPageAutomationToolName;
 
+export declare type JsonSchema = {
+    type?: string | string[];
+    enum?: readonly unknown[];
+    const?: unknown;
+    minimum?: number;
+    maximum?: number;
+    description?: string;
+    properties?: Record<string, JsonSchema>;
+    required?: readonly string[];
+    items?: JsonSchema;
+    additionalProperties?: boolean | JsonSchema;
+};
+
 export { }