@midscene/core 0.26.5-beta-20250814095614.0 → 0.26.5-beta-20250814125155.0

package/dist/lib/yaml.js

@@ -0,0 +1,20 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+for(var __webpack_i__ in __webpack_exports__)exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=yaml.js.map

package/dist/lib/yaml.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml.js","sources":["webpack://@midscene/core/webpack/runtime/make_namespace_object"],"sourcesContent":["// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","Symbol","Object"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOC,UAA0BA,OAAO,WAAW,EACrDC,OAAO,cAAc,CAAC,UAASD,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEC,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/types/ai-model/action-executor.d.ts

@@ -0,0 +1,19 @@
+import type { ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions } from '../types';
+export declare class Executor {
+    name: string;
+    tasks: ExecutionTask[];
+    status: 'init' | 'pending' | 'running' | 'completed' | 'error';
+    onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+    constructor(name: string, options?: ExecutionTaskProgressOptions & {
+        tasks?: ExecutionTaskApply[];
+    });
+    private markTaskAsPending;
+    append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
+    flush(): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): ExecutionTask | null;
+    dump(): ExecutionDump;
+}

package/dist/types/ai-model/common.d.ts

@@ -0,0 +1,34 @@
+import type { AIUsageInfo, BaseElement, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
+import type { PlanningLocateParam } from '../types';
+export type AIArgs = [
+    ChatCompletionSystemMessageParam,
+    ...ChatCompletionUserMessageParam[]
+];
+export declare enum AIActionType {
+    ASSERT = 0,
+    INSPECT_ELEMENT = 1,
+    EXTRACT_DATA = 2,
+    PLAN = 3,
+    DESCRIBE_ELEMENT = 4
+}
+export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+    content: T;
+    usage?: AIUsageInfo;
+}>;
+export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number): PlanningLocateParam;
+export declare function adaptQwenBbox(bbox: number[]): [number, number, number, number];
+export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
+export declare function adaptBbox(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number): Rect;
+export declare function warnGPT4oSizeLimit(size: Size): void;
+export declare function mergeRects(rects: Rect[]): {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
+export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], sleep?: number): MidsceneYamlFlowItem[];

package/dist/types/ai-model/index.d.ts

@@ -0,0 +1,11 @@
+export { callAiFnWithStringResponse, callToGetJSONObject, call as callAi, } from './service-caller/index';
+export { systemPromptToLocateElement } from './prompt/llm-locator';
+export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
+export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
+export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
+export type { ChatCompletionMessageParam } from 'openai/resources/index';
+export { AiLocateElement, AiExtractElementInfo, AiAssert, AiLocateSection, } from './inspect';
+export { plan } from './llm-planning';
+export { callAiFn, adaptBboxToRect, } from './common';
+export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
+export { AIActionType, type AIArgs } from './common';

package/dist/types/ai-model/inspect.d.ts

@@ -0,0 +1,49 @@
+import type { AIAssertionResponse, AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
+import { callAiFn } from './common';
+export type AIArgs = [
+    ChatCompletionSystemMessageParam,
+    ...ChatCompletionUserMessageParam[]
+];
+export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
+    context: UIContext<ElementType>;
+    targetElementDescription: TUserPrompt;
+    referenceImage?: ReferenceImage;
+    callAI?: typeof callAiFn<AIElementResponse | [number, number]>;
+    searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
+}): Promise<{
+    parseResult: AIElementLocatorResponse;
+    rect?: Rect;
+    rawResponse: string;
+    elementById: ElementById;
+    usage?: AIUsageInfo;
+    isOrderSensitive?: boolean;
+}>;
+export declare function AiLocateSection(options: {
+    context: UIContext<BaseElement>;
+    sectionDescription: TUserPrompt;
+    callAI?: typeof callAiFn<AISectionLocatorResponse>;
+}): Promise<{
+    rect?: Rect;
+    imageBase64?: string;
+    error?: string;
+    rawResponse: string;
+    usage?: AIUsageInfo;
+}>;
+export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+    dataQuery: string | Record<string, string>;
+    multimodalPrompt?: TMultimodalPrompt;
+    context: UIContext<ElementType>;
+    extractOption?: InsightExtractOption;
+}): Promise<{
+    parseResult: AIDataExtractionResponse<T>;
+    elementById: (idOrIndexId: string) => ElementType;
+    usage: AIUsageInfo | undefined;
+}>;
+export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
+    assertion: TUserPrompt;
+    context: UIContext<ElementType>;
+}): Promise<{
+    content: AIAssertionResponse;
+    usage: AIUsageInfo | undefined;
+}>;

package/dist/types/ai-model/llm-planning.d.ts

@@ -0,0 +1,10 @@
+import type { DeviceAction, PageType, PlanningAIResponse, UIContext } from '../types';
+import { callAiFn } from './common';
+export declare function plan(userInstruction: string, opts: {
+    context: UIContext;
+    pageType: PageType;
+    actionSpace: DeviceAction[];
+    callAI?: typeof callAiFn<PlanningAIResponse>;
+    log?: string;
+    actionContext?: string;
+}): Promise<PlanningAIResponse>;

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -0,0 +1,5 @@
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare function systemPromptToAssert(model: {
+    isUITars: boolean;
+}): string;
+export declare const assertSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/common.d.ts

@@ -0,0 +1,2 @@
+import type { vlLocateMode } from '@midscene/shared/env';
+export declare function bboxDescription(vlMode: ReturnType<typeof vlLocateMode>): "2d bounding box as [ymin, xmin, ymax, xmax]" | "2d bounding box as [xmin, ymin, xmax, ymax]";

package/dist/types/ai-model/prompt/describe.d.ts

@@ -0,0 +1 @@
+export declare const elementDescriberInstruction: () => string;

package/dist/types/ai-model/prompt/extraction.d.ts

@@ -0,0 +1,4 @@
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare function systemPromptToExtract(): string;
+export declare const extractDataQueryPrompt: (pageDescription: string, dataQuery: string | Record<string, string>) => Promise<string>;
+export declare const extractDataSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/llm-locator.d.ts

@@ -0,0 +1,9 @@
+import { PromptTemplate } from '@langchain/core/prompts';
+import type { vlLocateMode } from '@midscene/shared/env';
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare function systemPromptToLocateElement(vlMode: ReturnType<typeof vlLocateMode>): string;
+export declare const locatorSchema: ResponseFormatJSONSchema;
+export declare const findElementPrompt: PromptTemplate<{
+    pageDescription: any;
+    targetElementDescription: any;
+}, any>;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -0,0 +1,15 @@
+import type { DeviceAction } from '../../types';
+import { PromptTemplate } from '@langchain/core/prompts';
+import type { vlLocateMode } from '@midscene/shared/env';
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare const descriptionForAction: (action: DeviceAction, locatorScheme: string) => string;
+export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, }: {
+    actionSpace: DeviceAction[];
+    vlMode: ReturnType<typeof vlLocateMode>;
+}): Promise<string>;
+export declare const planSchema: ResponseFormatJSONSchema;
+export declare const generateTaskBackgroundContext: (userInstruction: string, log?: string, userActionContext?: string) => string;
+export declare const automationUserPrompt: (vlMode: ReturnType<typeof vlLocateMode>) => PromptTemplate<{
+    pageDescription: any;
+    taskBackgroundContext: any;
+}, any>;

package/dist/types/ai-model/prompt/llm-section-locator.d.ts

@@ -0,0 +1,6 @@
+import { PromptTemplate } from '@langchain/core/prompts';
+import type { vlLocateMode } from '@midscene/shared/env';
+export declare function systemPromptToLocateSection(vlMode: ReturnType<typeof vlLocateMode>): string;
+export declare const sectionLocatorInstruction: PromptTemplate<{
+    sectionDescription: any;
+}, any>;

package/dist/types/ai-model/prompt/playwright-generator.d.ts

@@ -0,0 +1,25 @@
+import type { StreamingAIResponse, StreamingCodeGenerationOptions } from '../../types';
+import { type ChromeRecordedEvent, type EventCounts, type EventSummary, type InputDescription, type ProcessedEvent, createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents } from './yaml-generator';
+export interface PlaywrightGenerationOptions {
+    testName?: string;
+    includeScreenshots?: boolean;
+    includeTimestamps?: boolean;
+    maxScreenshots?: number;
+    description?: string;
+    viewportSize?: {
+        width: number;
+        height: number;
+    };
+    waitForNetworkIdle?: boolean;
+    waitForNetworkIdleTimeout?: number;
+}
+export type { ChromeRecordedEvent, EventCounts, InputDescription, ProcessedEvent, EventSummary, };
+export { getScreenshotsForLLM, filterEventsByType, createEventCounts, extractInputDescriptions, processEventsForLLM, prepareEventSummary, createMessageContent, validateEvents, };
+/**
+ * Generates Playwright test code from recorded events
+ */
+export declare const generatePlaywrightTest: (events: ChromeRecordedEvent[], options?: PlaywrightGenerationOptions) => Promise<string>;
+/**
+ * Generates Playwright test code from recorded events with streaming support
+ */
+export declare const generatePlaywrightTestStream: (events: ChromeRecordedEvent[], options?: PlaywrightGenerationOptions & StreamingCodeGenerationOptions) => Promise<StreamingAIResponse>;

package/dist/types/ai-model/prompt/ui-tars-locator.d.ts

@@ -0,0 +1 @@
+export declare function systemPromptToLocateElementPosition(): string;

package/dist/types/ai-model/prompt/ui-tars-planning.d.ts

@@ -0,0 +1,2 @@
+export declare function getUiTarsPlanningPrompt(): string;
+export declare const getSummary: (prediction: string) => string;

package/dist/types/ai-model/prompt/util.d.ts

@@ -0,0 +1,45 @@
+import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
+export declare function describeSize(size: Size): string;
+export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
+    id: string;
+})[]): string;
+export declare const distanceThreshold = 16;
+export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
+    x: number;
+    y: number;
+}, options?: {
+    requireStrictDistance?: boolean;
+    filterPositionElements?: boolean;
+}): BaseElement | undefined;
+export declare function distance(point1: {
+    x: number;
+    y: number;
+}, point2: {
+    x: number;
+    y: number;
+}): number;
+export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
+export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt?: {
+    truncateTextLength?: number;
+    filterNonTextContent?: boolean;
+    domIncluded?: boolean | 'visible-only';
+    visibleOnly?: boolean;
+}): Promise<{
+    description: string;
+    elementById(idOrIndexId: string): ElementType;
+    elementByPosition(position: {
+        x: number;
+        y: number;
+    }, size: {
+        width: number;
+        height: number;
+    }): BaseElement | undefined;
+    insertElementByPosition(position: {
+        x: number;
+        y: number;
+    }): ElementType;
+    size: {
+        width: number;
+        height: number;
+    };
+}>;

package/dist/types/ai-model/prompt/yaml-generator.d.ts

@@ -0,0 +1,99 @@
+import type { StreamingAIResponse, StreamingCodeGenerationOptions } from '../../types';
+export interface EventCounts {
+    navigation: number;
+    click: number;
+    input: number;
+    scroll: number;
+    total: number;
+}
+export interface InputDescription {
+    description: string;
+    value: string;
+}
+export interface ProcessedEvent {
+    type: string;
+    timestamp: number;
+    url?: string;
+    title?: string;
+    elementDescription?: string;
+    value?: string;
+    pageInfo?: any;
+    elementRect?: any;
+}
+export interface EventSummary {
+    testName: string;
+    startUrl: string;
+    eventCounts: EventCounts;
+    urls: string[];
+    clickDescriptions: string[];
+    inputDescriptions: InputDescription[];
+    events: ProcessedEvent[];
+}
+export interface ChromeRecordedEvent {
+    type: string;
+    timestamp: number;
+    url?: string;
+    title?: string;
+    elementDescription?: string;
+    value?: string;
+    pageInfo?: any;
+    elementRect?: any;
+    screenshotBefore?: string;
+    screenshotAfter?: string;
+    screenshotWithBox?: string;
+}
+export interface YamlGenerationOptions {
+    testName?: string;
+    includeTimestamps?: boolean;
+    maxScreenshots?: number;
+    description?: string;
+}
+export interface FilteredEvents {
+    navigationEvents: ChromeRecordedEvent[];
+    clickEvents: ChromeRecordedEvent[];
+    inputEvents: ChromeRecordedEvent[];
+    scrollEvents: ChromeRecordedEvent[];
+}
+/**
+ * Get screenshots from events for LLM context
+ */
+export declare const getScreenshotsForLLM: (events: ChromeRecordedEvent[], maxScreenshots?: number) => string[];
+/**
+ * Filter events by type for easier processing
+ */
+export declare const filterEventsByType: (events: ChromeRecordedEvent[]) => FilteredEvents;
+/**
+ * Create event counts summary
+ */
+export declare const createEventCounts: (filteredEvents: FilteredEvents, totalEvents: number) => EventCounts;
+/**
+ * Extract input descriptions from input events
+ */
+export declare const extractInputDescriptions: (inputEvents: ChromeRecordedEvent[]) => InputDescription[];
+/**
+ * Process events for LLM consumption
+ */
+export declare const processEventsForLLM: (events: ChromeRecordedEvent[]) => ProcessedEvent[];
+/**
+ * Prepare comprehensive event summary for LLM
+ */
+export declare const prepareEventSummary: (events: ChromeRecordedEvent[], options?: {
+    testName?: string;
+    maxScreenshots?: number;
+}) => EventSummary;
+/**
+ * Create message content for LLM with optional screenshots
+ */
+export declare const createMessageContent: (promptText: string, screenshots?: string[], includeScreenshots?: boolean) => any[];
+/**
+ * Validate events before processing
+ */
+export declare const validateEvents: (events: ChromeRecordedEvent[]) => void;
+/**
+ * Generates YAML test configuration from recorded events using AI
+ */
+export declare const generateYamlTest: (events: ChromeRecordedEvent[], options?: YamlGenerationOptions) => Promise<string>;
+/**
+ * Generates YAML test configuration from recorded events using AI with streaming support
+ */
+export declare const generateYamlTestStream: (events: ChromeRecordedEvent[], options?: YamlGenerationOptions & StreamingCodeGenerationOptions) => Promise<StreamingAIResponse>;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -0,0 +1,26 @@
+import { type AIUsageInfo } from '../../types';
+import type { StreamingCallback } from '../../types';
+import OpenAI from 'openai';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import { AIActionType, type AIArgs } from '../common';
+export declare function checkAIConfig(): boolean;
+export declare function getModelName(): string;
+export declare function call(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, responseFormat?: OpenAI.ChatCompletionCreateParams['response_format'] | OpenAI.ResponseFormatJSONObject, options?: {
+    stream?: boolean;
+    onChunk?: StreamingCallback;
+}): Promise<{
+    content: string;
+    usage?: AIUsageInfo;
+    isStreamed: boolean;
+}>;
+export declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
+    content: T;
+    usage?: AIUsageInfo;
+}>;
+export declare function callAiFnWithStringResponse<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+    content: string;
+    usage?: AIUsageInfo;
+}>;
+export declare function extractJSONFromCodeBlock(response: string): string;
+export declare function preprocessDoubaoBboxJson(input: string): string;
+export declare function safeParseJson(input: string): any;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -0,0 +1,76 @@
+import type { AIUsageInfo, MidsceneYamlFlowItem, PlanningAction, Size } from '../types';
+import { actionParser } from '@ui-tars/action-parser';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+type ActionType = 'click' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait' | 'androidBackButton' | 'androidHomeButton' | 'androidRecentAppsButton' | 'androidLongPress' | 'androidPull';
+export declare function vlmPlanning(options: {
+    userInstruction: string;
+    conversationHistory: ChatCompletionMessageParam[];
+    size: {
+        width: number;
+        height: number;
+    };
+}): Promise<{
+    actions: PlanningAction<any>[];
+    actionsFromModel: ReturnType<typeof actionParser>['parsed'];
+    action_summary: string;
+    yamlFlow?: MidsceneYamlFlowItem[];
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+}>;
+interface BaseAction {
+    action_type: ActionType;
+    action_inputs: Record<string, any>;
+    reflection: string | null;
+    thought: string | null;
+}
+interface ClickAction extends BaseAction {
+    action_type: 'click';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface DragAction extends BaseAction {
+    action_type: 'drag';
+    action_inputs: {
+        start_box: string;
+        end_box: string;
+    };
+}
+interface WaitAction extends BaseAction {
+    action_type: 'wait';
+    action_inputs: {
+        time: string;
+    };
+}
+interface TypeAction extends BaseAction {
+    action_type: 'type';
+    action_inputs: {
+        content: string;
+    };
+}
+interface HotkeyAction extends BaseAction {
+    action_type: 'hotkey';
+    action_inputs: {
+        key: string;
+    };
+}
+interface ScrollAction extends BaseAction {
+    action_type: 'scroll';
+    action_inputs: {
+        direction: 'up' | 'down';
+    };
+}
+interface FinishedAction extends BaseAction {
+    action_type: 'finished';
+    action_inputs: Record<string, never>;
+}
+interface AndroidLongPressAction extends BaseAction {
+    action_type: 'androidLongPress';
+    action_inputs: {
+        start_coords: [number, number];
+        duration?: number;
+    };
+}
+export type Action = ClickAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction | AndroidLongPressAction;
+export declare function resizeImageForUiTars(imageBase64: string, size: Size): Promise<string>;
+export {};

package/dist/types/image/index.d.ts

@@ -0,0 +1 @@
+export { imageInfo, imageInfoOfBase64, localImg2Base64, httpImg2Base64, resizeImg, saveBase64Image, zoomForGPT4o, } from '@midscene/shared/img';