@aiscene/core 8.0.1 → 8.0.3

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,33 +0,0 @@
-import type { SubGoal } from '../../types';
-/**
- * Extract content from an XML tag in a string, searching from the end.
- * This approach handles cases where models prepend thinking content (like <think>...</think>)
- * before the actual response tags, or when there are incomplete/nested tags.
- *
- * Strategy: Find the LAST closing tag, then search backwards for the nearest opening tag.
- * This ensures we get the last complete tag pair, even if there are incomplete tags before it.
- *
- * @param xmlString - The XML string to parse
- * @param tagName - The name of the tag to extract (case-insensitive)
- * @returns The trimmed content of the tag, or undefined if not found
- */
-export declare function extractXMLTag(xmlString: string, tagName: string): string | undefined;
-/**
- * Parse sub-goals from XML content
- * Handles both formats:
- * - <sub-goal index="1" status="pending">description</sub-goal>
- * - <sub-goal index="1" status="finished" />
- */
-export declare function parseSubGoalsFromXML(xmlContent: string): SubGoal[];
-/**
- * Extract indexes of sub-goals marked as finished from <mark-sub-goal-done> content
- */
-export declare function parseMarkFinishedIndexes(xmlContent: string): number[];
-export declare const distanceThreshold = 16;
-export declare function distance(point1: {
-    x: number;
-    y: number;
-}, point2: {
-    x: number;
-    y: number;
-}): number;

package/dist/types/ai-model/prompt/yaml-generator.d.ts

@@ -1,102 +0,0 @@
-import type { StreamingAIResponse, StreamingCodeGenerationOptions } from '../../types';
-import type { IModelConfig } from '@midscene/shared/env';
-export interface EventCounts {
-    navigation: number;
-    click: number;
-    input: number;
-    scroll: number;
-    total: number;
-}
-export interface InputDescription {
-    description: string;
-    value: string;
-}
-export interface ProcessedEvent {
-    type: string;
-    timestamp: number;
-    url?: string;
-    title?: string;
-    elementDescription?: string;
-    value?: string;
-    pageInfo?: any;
-    elementRect?: any;
-}
-export interface EventSummary {
-    testName: string;
-    startUrl: string;
-    eventCounts: EventCounts;
-    urls: string[];
-    clickDescriptions: string[];
-    inputDescriptions: InputDescription[];
-    events: ProcessedEvent[];
-}
-export interface ChromeRecordedEvent {
-    type: string;
-    timestamp: number;
-    url?: string;
-    title?: string;
-    elementDescription?: string;
-    value?: string;
-    pageInfo?: any;
-    elementRect?: any;
-    screenshotBefore?: string;
-    screenshotAfter?: string;
-    screenshotWithBox?: string;
-}
-export interface YamlGenerationOptions {
-    testName?: string;
-    includeTimestamps?: boolean;
-    maxScreenshots?: number;
-    description?: string;
-    /** Language for human-readable YAML content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */
-    language?: string;
-}
-export interface FilteredEvents {
-    navigationEvents: ChromeRecordedEvent[];
-    clickEvents: ChromeRecordedEvent[];
-    inputEvents: ChromeRecordedEvent[];
-    scrollEvents: ChromeRecordedEvent[];
-}
-/**
- * Get screenshots from events for LLM context
- */
-export declare const getScreenshotsForLLM: (events: ChromeRecordedEvent[], maxScreenshots?: number) => string[];
-/**
- * Filter events by type for easier processing
- */
-export declare const filterEventsByType: (events: ChromeRecordedEvent[]) => FilteredEvents;
-/**
- * Create event counts summary
- */
-export declare const createEventCounts: (filteredEvents: FilteredEvents, totalEvents: number) => EventCounts;
-/**
- * Extract input descriptions from input events
- */
-export declare const extractInputDescriptions: (inputEvents: ChromeRecordedEvent[]) => InputDescription[];
-/**
- * Process events for LLM consumption
- */
-export declare const processEventsForLLM: (events: ChromeRecordedEvent[]) => ProcessedEvent[];
-/**
- * Prepare comprehensive event summary for LLM
- */
-export declare const prepareEventSummary: (events: ChromeRecordedEvent[], options?: {
-    testName?: string;
-    maxScreenshots?: number;
-}) => EventSummary;
-/**
- * Create message content for LLM with optional screenshots
- */
-export declare const createMessageContent: (promptText: string, screenshots?: string[], includeScreenshots?: boolean) => any[];
-/**
- * Validate events before processing
- */
-export declare const validateEvents: (events: ChromeRecordedEvent[]) => void;
-/**
- * Generates YAML test configuration from recorded events using AI
- */
-export declare const generateYamlTest: (events: ChromeRecordedEvent[], options: YamlGenerationOptions, modelConfig: IModelConfig) => Promise<string>;
-/**
- * Generates YAML test configuration from recorded events using AI with streaming support
- */
-export declare const generateYamlTestStream: (events: ChromeRecordedEvent[], options: YamlGenerationOptions & StreamingCodeGenerationOptions, modelConfig: IModelConfig) => Promise<StreamingAIResponse>;

package/dist/types/ai-model/service-caller/codex-app-server.d.ts

@@ -1,42 +0,0 @@
-import type { AIUsageInfo, DeepThinkOption, StreamingCallback } from '../../types';
-import type { IModelConfig } from '@midscene/shared/env';
-import type { ChatCompletionMessageParam } from 'openai/resources/index';
-type CodexReasoningEffort = 'low' | 'medium' | 'high' | 'xhigh';
-type CodexTextInput = {
-    type: 'text';
-    text: string;
-    text_elements: any[];
-};
-type CodexImageInput = {
-    type: 'image';
-    url: string;
-};
-type CodexLocalImageInput = {
-    type: 'localImage';
-    path: string;
-};
-type CodexTurnInput = CodexTextInput | CodexImageInput | CodexLocalImageInput;
-type CodexTurnResult = {
-    content: string;
-    reasoning_content?: string;
-    usage?: AIUsageInfo;
-    isStreamed: boolean;
-};
-export declare const isCodexAppServerProvider: (baseURL?: string) => boolean;
-export declare const normalizeCodexLocalImagePath: (imageUrl: string, platform?: NodeJS.Platform) => string;
-export declare const resolveCodexReasoningEffort: ({ deepThink, modelConfig, }: {
-    deepThink?: DeepThinkOption;
-    modelConfig: IModelConfig;
-}) => CodexReasoningEffort | undefined;
-export declare const buildCodexTurnPayloadFromMessages: (messages: ChatCompletionMessageParam[]) => {
-    developerInstructions?: string;
-    input: CodexTurnInput[];
-};
-export declare function callAIWithCodexAppServer(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
-    stream?: boolean;
-    onChunk?: StreamingCallback;
-    deepThink?: DeepThinkOption;
-    abortSignal?: AbortSignal;
-}): Promise<CodexTurnResult>;
-export declare function __shutdownCodexAppServerForTests(): Promise<void>;
-export {};

package/dist/types/ai-model/service-caller/image-detail.d.ts

@@ -1,2 +0,0 @@
-import type { IModelConfig } from '@midscene/shared/env';
-export declare function shouldForceOriginalImageDetail(modelConfig: Pick<IModelConfig, 'intent' | 'modelFamily'>): boolean;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,49 +0,0 @@
-import type { AIUsageInfo, DeepThinkOption } from '../../types';
-import type { StreamingCallback } from '../../types';
-export declare class AIResponseParseError extends Error {
-    usage?: AIUsageInfo;
-    rawResponse: string;
-    constructor(message: string, rawResponse: string, usage?: AIUsageInfo);
-}
-import { type IModelConfig, type TModelFamily } from '@midscene/shared/env';
-import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import type { AIArgs } from '../../common';
-export declare function callAI(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
-    stream?: boolean;
-    onChunk?: StreamingCallback;
-    deepThink?: DeepThinkOption;
-    abortSignal?: AbortSignal;
-}): Promise<{
-    content: string;
-    reasoning_content?: string;
-    usage?: AIUsageInfo;
-    isStreamed: boolean;
-}>;
-export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
-    deepThink?: DeepThinkOption;
-    abortSignal?: AbortSignal;
-}): Promise<{
-    content: T;
-    contentString: string;
-    usage?: AIUsageInfo;
-    reasoning_content?: string;
-}>;
-export declare function callAIWithStringResponse(msgs: AIArgs, modelConfig: IModelConfig, options?: {
-    abortSignal?: AbortSignal;
-}): Promise<{
-    content: string;
-    usage?: AIUsageInfo;
-}>;
-export declare function extractJSONFromCodeBlock(response: string): string;
-export declare function preprocessDoubaoBboxJson(input: string): string;
-export declare function resolveReasoningConfig({ reasoningEnabled, reasoningEffort, reasoningBudget, modelFamily, }: {
-    reasoningEnabled?: boolean;
-    reasoningEffort?: string;
-    reasoningBudget?: number;
-    modelFamily?: TModelFamily;
-}): {
-    config: Record<string, unknown>;
-    debugMessage?: string;
-    warningMessage?: string;
-};
-export declare function safeParseJson(input: string, modelFamily: TModelFamily | undefined): any;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -1,72 +0,0 @@
-import type { PlanningAIResponse, UIContext } from '../types';
-import { type IModelConfig } from '@midscene/shared/env';
-import type { ConversationHistory } from './conversation-history';
-type ActionType = 'click' | 'left_double' | 'right_single' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
-export declare function uiTarsPlanning(userInstruction: string, options: {
-    conversationHistory: ConversationHistory;
-    context: UIContext;
-    modelConfig: IModelConfig;
-    actionContext?: string;
-    abortSignal?: AbortSignal;
-}): Promise<PlanningAIResponse>;
-interface BaseAction {
-    action_type: ActionType;
-    action_inputs: Record<string, any>;
-    reflection: string | null;
-    thought: string | null;
-}
-interface ClickAction extends BaseAction {
-    action_type: 'click';
-    action_inputs: {
-        start_box: string;
-    };
-}
-interface DragAction extends BaseAction {
-    action_type: 'drag';
-    action_inputs: {
-        start_box: string;
-        end_box: string;
-    };
-}
-interface WaitAction extends BaseAction {
-    action_type: 'wait';
-    action_inputs: {
-        time: string;
-    };
-}
-interface LeftDoubleAction extends BaseAction {
-    action_type: 'left_double';
-    action_inputs: {
-        start_box: string;
-    };
-}
-interface RightSingleAction extends BaseAction {
-    action_type: 'right_single';
-    action_inputs: {
-        start_box: string;
-    };
-}
-interface TypeAction extends BaseAction {
-    action_type: 'type';
-    action_inputs: {
-        content: string;
-    };
-}
-interface HotkeyAction extends BaseAction {
-    action_type: 'hotkey';
-    action_inputs: {
-        key: string;
-    };
-}
-interface ScrollAction extends BaseAction {
-    action_type: 'scroll';
-    action_inputs: {
-        direction: 'up' | 'down';
-    };
-}
-interface FinishedAction extends BaseAction {
-    action_type: 'finished';
-    action_inputs: Record<string, never>;
-}
-export type Action = ClickAction | LeftDoubleAction | RightSingleAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
-export {};

package/dist/types/common.d.ts

@@ -1,288 +0,0 @@
-import type { BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from './types';
-import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import type { PlanningLocateParam } from './types';
-import type { TModelFamily } from '@midscene/shared/env';
-import { z } from 'zod';
-export type AIArgs = ChatCompletionMessageParam[];
-type AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];
-/**
- * Convert a point coordinate [0, 1000] to a small bbox [0, 1000]
- * Creates a small bbox around the center point in the same coordinate space
- *
- * @param x - X coordinate in [0, 1000] range
- * @param y - Y coordinate in [0, 1000] range
- * @param bboxSize - Size of the bbox to create (default: 20)
- * @returns [x1, y1, x2, y2] bbox in [0, 1000] coordinate space
- */
-export declare function pointToBbox(x: number, y: number, bboxSize?: number): [number, number, number, number];
-export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, modelFamily: TModelFamily | undefined): PlanningLocateParam;
-export declare function adaptQwen2_5Bbox(bbox: number[]): [number, number, number, number];
-export declare function adaptGpt5Bbox(bbox: number[] | string[] | string): [number, number, number, number];
-export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
-export declare function adaptBbox(bbox: AdaptBboxInput, width: number, height: number, modelFamily: TModelFamily | undefined): [number, number, number, number];
-export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
-export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
-export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, modelFamily?: TModelFamily | undefined, scale?: number): Rect;
-export declare function mergeRects(rects: Rect[]): {
-    left: number;
-    top: number;
-    width: number;
-    height: number;
-};
-/**
- * Expand the search area to at least 400 x 400 pixels
- *
- * Step 1: Extend 100px on each side (top, right, bottom, left)
- * - If the element is near a boundary, expansion on that side will be limited
- * - No compensation is made for boundary limitations (this is intentional)
- *
- * Step 2: Ensure the area is at least 400x400 pixels
- * - Scale up proportionally from the center if needed
- * - Final result is clamped to screen boundaries
- */
-export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
-export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
-export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[]): MidsceneYamlFlowItem[];
-export declare const PointSchema: z.ZodObject<{
-    left: z.ZodNumber;
-    top: z.ZodNumber;
-}, "strip", z.ZodTypeAny, {
-    left: number;
-    top: number;
-}, {
-    left: number;
-    top: number;
-}>;
-export declare const SizeSchema: z.ZodObject<{
-    width: z.ZodNumber;
-    height: z.ZodNumber;
-}, "strip", z.ZodTypeAny, {
-    width: number;
-    height: number;
-}, {
-    width: number;
-    height: number;
-}>;
-export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
-    left: z.ZodNumber;
-    top: z.ZodNumber;
-}, "strip", z.ZodTypeAny, {
-    left: number;
-    top: number;
-}, {
-    left: number;
-    top: number;
-}>, z.ZodObject<{
-    width: z.ZodNumber;
-    height: z.ZodNumber;
-}, "strip", z.ZodTypeAny, {
-    width: number;
-    height: number;
-}, {
-    width: number;
-    height: number;
-}>>, z.ZodObject<{
-    zoom: z.ZodOptional<z.ZodNumber>;
-}, "strip", z.ZodTypeAny, {
-    zoom?: number | undefined;
-}, {
-    zoom?: number | undefined;
-}>>;
-export declare const TMultimodalPromptSchema: z.ZodObject<{
-    images: z.ZodOptional<z.ZodArray<z.ZodObject<{
-        name: z.ZodString;
-        url: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        name: string;
-        url: string;
-    }, {
-        name: string;
-        url: string;
-    }>, "many">>;
-    convertHttpImage2Base64: z.ZodOptional<z.ZodBoolean>;
-}, "strip", z.ZodTypeAny, {
-    images?: {
-        name: string;
-        url: string;
-    }[] | undefined;
-    convertHttpImage2Base64?: boolean | undefined;
-}, {
-    images?: {
-        name: string;
-        url: string;
-    }[] | undefined;
-    convertHttpImage2Base64?: boolean | undefined;
-}>;
-export declare const TUserPromptSchema: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-    prompt: z.ZodString;
-}, "strip", z.ZodTypeAny, {
-    prompt: string;
-}, {
-    prompt: string;
-}>, z.ZodObject<{
-    images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-        name: z.ZodString;
-        url: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        name: string;
-        url: string;
-    }, {
-        name: string;
-        url: string;
-    }>, "many">>>;
-    convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-}, "strip", z.ZodTypeAny, {
-    images?: {
-        name: string;
-        url: string;
-    }[] | undefined;
-    convertHttpImage2Base64?: boolean | undefined;
-}, {
-    images?: {
-        name: string;
-        url: string;
-    }[] | undefined;
-    convertHttpImage2Base64?: boolean | undefined;
-}>>]>;
-export type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;
-export type TUserPrompt = z.infer<typeof TUserPromptSchema>;
-/**
- * Returns the schema for locator fields.
- * This now returns the input schema which is more permissive and suitable for validation.
- */
-export declare const getMidsceneLocationSchema: () => z.ZodObject<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepLocate: z.ZodOptional<z.ZodBoolean>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepLocate: z.ZodOptional<z.ZodBoolean>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepLocate: z.ZodOptional<z.ZodBoolean>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, z.ZodTypeAny, "passthrough">>;
-export declare const ifMidsceneLocatorField: (field: any) => boolean;
-export declare const dumpMidsceneLocatorField: (field: any) => string;
-export declare const findAllMidsceneLocatorField: (zodType?: z.ZodType<any>, requiredOnly?: boolean) => string[];
-export declare const dumpActionParam: (jsonObject: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
-/**
- * Parse and validate action parameters using Zod schema.
- * All fields are validated through Zod, EXCEPT locator fields which are skipped.
- * Default values defined in the schema are automatically applied.
- *
- * Locator fields are special business logic fields with complex validation requirements,
- * so they are intentionally excluded from Zod parsing and use existing validation logic.
- *
- * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields
- * are transformed from screenshot space to logical space.
- */
-export declare const parseActionParam: (rawParam: Record<string, any> | undefined, zodSchema?: z.ZodType<any>, options?: {
-    shrunkShotToLogicalRatio?: number;
-}) => Record<string, any> | undefined;
-export declare const finalizeActionName = "Finalize";
-/**
- * Get a readable time string for a given timestamp or the current time
- * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'
- * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.
- * @returns A formatted time string with format label
- */
-export declare const getReadableTimeString: (format?: string, timestamp?: number) => string;
-export {};