@rpascene/core 0.30.8

package/dist/types/ai-model/common.d.ts

@@ -0,0 +1,569 @@
+import type { BaseElement, DeviceAction, ElementTreeNode, RpasceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { PlanningLocateParam } from '../types';
+import type { TVlModeTypes } from '@rpascene/shared/env';
+import { z } from 'zod';
+export type AIArgs = ChatCompletionMessageParam[];
+export declare enum AIActionType {
+    ASSERT = 0,
+    INSPECT_ELEMENT = 1,
+    EXTRACT_DATA = 2,
+    PLAN = 3,
+    DESCRIBE_ELEMENT = 4,
+    TEXT = 5
+}
+export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): PlanningLocateParam;
+export declare function adaptQwenBbox(bbox: number[]): [number, number, number, number];
+export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
+export declare function adaptBbox(bbox: number[], width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): [number, number, number, number];
+export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, vlMode?: TVlModeTypes | undefined): Rect;
+export declare function warnGPT4oSizeLimit(size: Size, modelName: string): void;
+export declare function mergeRects(rects: Rect[]): {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+export declare function expandSearchArea(rect: Rect, screenSize: Size, vlMode: TVlModeTypes | undefined): Rect;
+export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[], sleep?: number): RpasceneYamlFlowItem[];
+export declare const PointSchema: z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>;
+export declare const SizeSchema: z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+    dpr: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}>;
+export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>, z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+    dpr: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}>>, z.ZodObject<{
+    zoom: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    zoom?: number | undefined;
+}, {
+    zoom?: number | undefined;
+}>>;
+export declare const TMultimodalPromptSchema: z.ZodObject<{
+    images: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        url: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        url: string;
+    }, {
+        name: string;
+        url: string;
+    }>, "many">>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}>;
+export declare const TUserPromptSchema: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+    prompt: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    prompt: string;
+}, {
+    prompt: string;
+}>, z.ZodObject<{
+    images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        url: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        url: string;
+    }, {
+        name: string;
+        url: string;
+    }>, "many">>>;
+    convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+}, "strip", z.ZodTypeAny, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}, {
+    images?: {
+        name: string;
+        url: string;
+    }[] | undefined;
+    convertHttpImage2Base64?: boolean | undefined;
+}>>]>;
+export type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;
+export type TUserPrompt = z.infer<typeof TUserPromptSchema>;
+declare const RpasceneLocationInput: z.ZodObject<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">>;
+declare const RpasceneLocationResult: z.ZodObject<{
+    rpascene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodBoolean>;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    rpascene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodBoolean>;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    rpascene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodBoolean>;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, z.ZodTypeAny, "passthrough">>;
+export type RpasceneLocationResultType = z.infer<typeof RpasceneLocationResult>;
+export type RpasceneLocationInputType = z.infer<typeof RpasceneLocationInput>;
+/**
+ * Returns the schema for locator fields.
+ * This now returns the input schema which is more permissive and suitable for validation.
+ */
+export declare const getRpasceneLocationSchema: () => z.ZodObject<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
+        prompt: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        prompt: string;
+    }, {
+        prompt: string;
+    }>, z.ZodObject<{
+        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            url: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            url: string;
+        }, {
+            name: string;
+            url: string;
+        }>, "many">>>;
+        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }, {
+        images?: {
+            name: string;
+            url: string;
+        }[] | undefined;
+        convertHttpImage2Base64?: boolean | undefined;
+    }>>]>;
+    deepThink: z.ZodOptional<z.ZodBoolean>;
+    cacheable: z.ZodOptional<z.ZodBoolean>;
+    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
+}, z.ZodTypeAny, "passthrough">>;
+export declare const ifRpasceneLocatorField: (field: any) => boolean;
+export declare const dumpRpasceneLocatorField: (field: any) => string;
+export declare const findAllRpasceneLocatorField: (zodType?: z.ZodType<any>, requiredOnly?: boolean) => string[];
+export declare const dumpActionParam: (jsonObject: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
+export declare const loadActionParam: (jsonObject: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
+/**
+ * Parse and validate action parameters using Zod schema.
+ * All fields are validated through Zod, EXCEPT locator fields which are skipped.
+ * Default values defined in the schema are automatically applied.
+ *
+ * Locator fields are special business logic fields with complex validation requirements,
+ * so they are intentionally excluded from Zod parsing and use existing validation logic.
+ */
+export declare const parseActionParam: (rawParam: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
+export {};

package/dist/types/ai-model/conversation-history.d.ts

@@ -0,0 +1,18 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+export interface ConversationHistoryOptions {
+    maxUserImageMessages?: number;
+    initialMessages?: ChatCompletionMessageParam[];
+}
+export declare class ConversationHistory {
+    private readonly maxUserImageMessages;
+    private readonly messages;
+    constructor(options?: ConversationHistoryOptions);
+    append(message: ChatCompletionMessageParam): void;
+    seed(messages: ChatCompletionMessageParam[]): void;
+    reset(): void;
+    snapshot(): ChatCompletionMessageParam[];
+    get length(): number;
+    [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam>;
+    toJSON(): ChatCompletionMessageParam[];
+    private pruneOldestUserMessageIfNecessary;
+}

package/dist/types/ai-model/index.d.ts

@@ -0,0 +1,13 @@
+export { callAIWithStringResponse, callAIWithObjectResponse, callAI, } from './service-caller/index';
+export { systemPromptToLocateElement } from './prompt/llm-locator';
+export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
+export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
+export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
+export type { ChatCompletionMessageParam } from 'openai/resources/index';
+export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
+export { plan } from './llm-planning';
+export { adaptBboxToRect } from './common';
+export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
+export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
+export { AIActionType, type AIArgs } from './common';
+export { getRpasceneLocationSchema, type RpasceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllRpasceneLocatorField, dumpActionParam, loadActionParam, parseActionParam, } from './common';

package/dist/types/ai-model/inspect.d.ts

@@ -0,0 +1,46 @@
+import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
+import type { IModelConfig } from '@rpascene/shared/env';
+import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
+import type { TMultimodalPrompt, TUserPrompt } from './common';
+import { callAIWithObjectResponse } from './service-caller/index';
+export type AIArgs = [
+    ChatCompletionSystemMessageParam,
+    ...ChatCompletionUserMessageParam[]
+];
+export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
+    context: UIContext<ElementType>;
+    targetElementDescription: TUserPrompt;
+    referenceImage?: ReferenceImage;
+    callAIFn: typeof callAIWithObjectResponse<AIElementResponse | [number, number]>;
+    searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
+    modelConfig: IModelConfig;
+}): Promise<{
+    parseResult: AIElementLocatorResponse;
+    rect?: Rect;
+    rawResponse: string;
+    elementById: ElementById;
+    usage?: AIUsageInfo;
+    isOrderSensitive?: boolean;
+}>;
+export declare function AiLocateSection(options: {
+    context: UIContext<BaseElement>;
+    sectionDescription: TUserPrompt;
+    modelConfig: IModelConfig;
+}): Promise<{
+    rect?: Rect;
+    imageBase64?: string;
+    error?: string;
+    rawResponse: string;
+    usage?: AIUsageInfo;
+}>;
+export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+    dataQuery: string | Record<string, string>;
+    multimodalPrompt?: TMultimodalPrompt;
+    context: UIContext<ElementType>;
+    extractOption?: InsightExtractOption;
+    modelConfig: IModelConfig;
+}): Promise<{
+    parseResult: AIDataExtractionResponse<T>;
+    elementById: (idOrIndexId: string) => ElementType;
+    usage: AIUsageInfo | undefined;
+}>;

package/dist/types/ai-model/llm-planning.d.ts

@@ -0,0 +1,11 @@
+import type { DeviceAction, InterfaceType, PlanningAIResponse, UIContext } from '../types';
+import type { IModelConfig } from '@rpascene/shared/env';
+import type { ConversationHistory } from './conversation-history';
+export declare function plan(userInstruction: string, opts: {
+    context: UIContext;
+    interfaceType: InterfaceType;
+    actionSpace: DeviceAction<any>[];
+    actionContext?: string;
+    modelConfig: IModelConfig;
+    conversationHistory?: ConversationHistory;
+}): Promise<PlanningAIResponse>;

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -0,0 +1,2 @@
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare const assertSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/common.d.ts

@@ -0,0 +1,2 @@
+import type { TVlModeTypes } from '@rpascene/shared/env';
+export declare function bboxDescription(vlMode: TVlModeTypes | undefined): "box_2d bounding box for the target element, should be [ymin, xmin, ymax, xmax] normalized to 0-1000." | "2d bounding box as [xmin, ymin, xmax, ymax]";

package/dist/types/ai-model/prompt/describe.d.ts

@@ -0,0 +1 @@
+export declare const elementDescriberInstruction: () => string;

package/dist/types/ai-model/prompt/extraction.d.ts

@@ -0,0 +1,4 @@
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare function systemPromptToExtract(): string;
+export declare const extractDataQueryPrompt: (pageDescription: string, dataQuery: string | Record<string, string>) => Promise<string>;
+export declare const extractDataSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/llm-locator.d.ts

@@ -0,0 +1,9 @@
+import { PromptTemplate } from '@langchain/core/prompts';
+import type { TVlModeTypes } from '@rpascene/shared/env';
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare function systemPromptToLocateElement(vlMode: TVlModeTypes | undefined): string;
+export declare const locatorSchema: ResponseFormatJSONSchema;
+export declare const findElementPrompt: PromptTemplate<{
+    pageDescription: any;
+    targetElementDescription: any;
+}, any>;