@midscene/core 1.4.6 → 1.4.7

package/dist/types/agent/agent.d.ts

@@ -41,14 +41,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * Flag to track if VL model warning has been shown
      */
     private hasWarnedNonVLModel;
-    /**
-     * Screenshot scale factor derived from actual screenshot dimensions
-     */
-    private screenshotScale?;
-    /**
-     * Internal promise to deduplicate screenshot scale computation
-     */
-    private screenshotScalePromise?;
     private executionDumpIndexByRunner;
     private fullActionSpace;
     private reportGenerator;
@@ -57,10 +49,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * Ensures VL model warning is shown once when needed
      */
     private ensureVLModelWarning;
-    /**
-     * Lazily compute the ratio between the physical screenshot width and the logical page width
-     */
-    private getScreenshotScale;
     private resolveReplanningCycleLimit;
     constructor(interfaceInstance: InterfaceType, opts?: AgentOpt);
     getActionSpace(): Promise<DeviceAction[]>;
@@ -132,9 +120,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
         deepThink?: boolean;
     } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
     verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
-    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "center" | "rect"> & {
-        dpr?: number;
-    }>;
+    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
     aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & ServiceExtractOption): Promise<{
         pass: boolean;
         thought: string | undefined;

package/dist/types/agent/utils.d.ts

@@ -1,6 +1,6 @@
 import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import type { AbstractInterface } from '../device';
-import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
+import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, Rect, UIContext } from '../types';
 import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
@@ -24,3 +24,15 @@ export declare const parsePrompt: (prompt: TUserPrompt) => {
     textPrompt: string;
     multimodalPrompt?: TMultimodalPrompt;
 };
+/**
+ * Transform coordinates from screenshot coordinate system to logical coordinate system.
+ * When shrunkShotToLogicalRatio > 1, the screenshot is larger than logical size,
+ * so we need to divide coordinates by shrunkShotToLogicalRatio.
+ *
+ * @param element - The locate result element with coordinates in screenshot space
+ * @param shrunkShotToLogicalRatio - The ratio of screenshot size to logical size
+ * @returns A new element with coordinates transformed to logical space
+ */
+export declare const transformScreenshotElementToLogical: (element: LocateResultElement, shrunkShotToLogicalRatio: number) => LocateResultElement;
+export declare const transformLogicalElementToScreenshot: (element: LocateResultElement, shrunkShotToLogicalRatio: number) => LocateResultElement;
+export declare const transformLogicalRectToScreenshotRect: (rect: Rect, shrunkShotToLogicalRatio: number) => Rect;

package/dist/types/ai-model/index.d.ts

@@ -11,4 +11,4 @@ export { uiTarsPlanning } from './ui-tars-planning';
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
 export type { SubGoal, SubGoalStatus } from '../types';
 export type { AIArgs } from '../common';
-export { getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, parseActionParam, } from '../common';
+export { getMidsceneLocationSchema, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, parseActionParam, } from '../common';

package/dist/types/ai-model/service-caller/index.d.ts

@@ -32,8 +32,10 @@ export declare function callAIWithStringResponse(msgs: AIArgs, modelConfig: IMod
 }>;
 export declare function extractJSONFromCodeBlock(response: string): string;
 export declare function preprocessDoubaoBboxJson(input: string): string;
-export declare function resolveDeepThinkConfig({ deepThink, modelFamily, }: {
-    deepThink?: DeepThinkOption;
+export declare function resolveReasoningConfig({ reasoningEnabled, reasoningEffort, reasoningBudget, modelFamily, }: {
+    reasoningEnabled?: boolean;
+    reasoningEffort?: string;
+    reasoningBudget?: number;
     modelFamily?: TModelFamily;
 }): {
     config: Record<string, unknown>;

package/dist/types/common.d.ts

@@ -55,15 +55,12 @@ export declare const PointSchema: z.ZodObject<{
 export declare const SizeSchema: z.ZodObject<{
     width: z.ZodNumber;
     height: z.ZodNumber;
-    dpr: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     width: number;
     height: number;
-    dpr?: number | undefined;
 }, {
     width: number;
     height: number;
-    dpr?: number | undefined;
 }>;
 export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
     left: z.ZodNumber;
@@ -77,15 +74,12 @@ export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject
 }>, z.ZodObject<{
     width: z.ZodNumber;
     height: z.ZodNumber;
-    dpr: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     width: number;
     height: number;
-    dpr?: number | undefined;
 }, {
     width: number;
     height: number;
-    dpr?: number | undefined;
 }>>, z.ZodObject<{
     zoom: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
@@ -151,310 +145,6 @@ export declare const TUserPromptSchema: z.ZodUnion<[z.ZodString, z.ZodIntersecti
 }>>]>;
 export type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;
 export type TUserPrompt = z.infer<typeof TUserPromptSchema>;
-declare const MidsceneLocationInput: z.ZodObject<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
-}, z.ZodTypeAny, "passthrough">>;
-declare const MidsceneLocationResult: z.ZodObject<{
-    midscene_location_field_flag: z.ZodLiteral<true>;
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodBoolean>;
-    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
-    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
-        left: z.ZodNumber;
-        top: z.ZodNumber;
-    }, "strip", z.ZodTypeAny, {
-        left: number;
-        top: number;
-    }, {
-        left: number;
-        top: number;
-    }>, z.ZodObject<{
-        width: z.ZodNumber;
-        height: z.ZodNumber;
-        dpr: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }>>, z.ZodObject<{
-        zoom: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        zoom?: number | undefined;
-    }, {
-        zoom?: number | undefined;
-    }>>;
-}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
-    midscene_location_field_flag: z.ZodLiteral<true>;
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodBoolean>;
-    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
-    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
-        left: z.ZodNumber;
-        top: z.ZodNumber;
-    }, "strip", z.ZodTypeAny, {
-        left: number;
-        top: number;
-    }, {
-        left: number;
-        top: number;
-    }>, z.ZodObject<{
-        width: z.ZodNumber;
-        height: z.ZodNumber;
-        dpr: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }>>, z.ZodObject<{
-        zoom: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        zoom?: number | undefined;
-    }, {
-        zoom?: number | undefined;
-    }>>;
-}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
-    midscene_location_field_flag: z.ZodLiteral<true>;
-    prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
-        prompt: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        prompt: string;
-    }, {
-        prompt: string;
-    }>, z.ZodObject<{
-        images: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
-            name: z.ZodString;
-            url: z.ZodString;
-        }, "strip", z.ZodTypeAny, {
-            name: string;
-            url: string;
-        }, {
-            name: string;
-            url: string;
-        }>, "many">>>;
-        convertHttpImage2Base64: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }, {
-        images?: {
-            name: string;
-            url: string;
-        }[] | undefined;
-        convertHttpImage2Base64?: boolean | undefined;
-    }>>]>;
-    deepThink: z.ZodOptional<z.ZodBoolean>;
-    cacheable: z.ZodOptional<z.ZodBoolean>;
-    xpath: z.ZodOptional<z.ZodBoolean>;
-    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
-    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
-        left: z.ZodNumber;
-        top: z.ZodNumber;
-    }, "strip", z.ZodTypeAny, {
-        left: number;
-        top: number;
-    }, {
-        left: number;
-        top: number;
-    }>, z.ZodObject<{
-        width: z.ZodNumber;
-        height: z.ZodNumber;
-        dpr: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }, {
-        width: number;
-        height: number;
-        dpr?: number | undefined;
-    }>>, z.ZodObject<{
-        zoom: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        zoom?: number | undefined;
-    }, {
-        zoom?: number | undefined;
-    }>>;
-}, z.ZodTypeAny, "passthrough">>;
-export type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;
-export type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;
 /**
  * Returns the schema for locator fields.
  * This now returns the input schema which is more permissive and suitable for validation.

package/dist/types/device/index.d.ts

@@ -2,7 +2,7 @@ import type { ActionScrollParam, DeviceAction, LocateResultElement } from '../ty
 import type { IModelConfig } from '@midscene/shared/env';
 import type { ElementNode } from '@midscene/shared/extractor';
 import { z } from 'zod';
-import type { ElementCacheFeature, Rect, Size, UIContext } from '../types';
+import type { ElementCacheFeature, Rect, Size } from '../types';
 export interface FileChooserHandler {
     accept(files: string[]): Promise<void>;
 }
@@ -27,7 +27,6 @@ export declare abstract class AbstractInterface {
     abstract getElementsNodeTree?: () => Promise<ElementNode>;
     abstract url?: () => string | Promise<string>;
     abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
-    abstract getContext?(): Promise<UIContext>;
     /**
      * Get the current time from the device.
      * Returns the device's current timestamp in milliseconds.

package/dist/types/dump/html-utils.d.ts

@@ -47,5 +47,16 @@ export declare function parseImageScripts(html: string): Record<string, string>;
 export declare function parseDumpScript(html: string): string;
 export declare function parseDumpScriptAttributes(html: string): Record<string, string>;
 export declare function generateImageScriptTag(id: string, data: string): string;
+/**
+ * Inline script that fixes relative URL resolution for directory-mode reports.
+ *
+ * Problem: when a static server (e.g. `npx serve`) serves `name/index.html`
+ * at URL `/name` (without trailing slash), relative paths like
+ * `./screenshots/xxx.png` resolve to `/screenshots/xxx.png` instead of
+ * `/name/screenshots/xxx.png`.
+ *
+ * Fix: dynamically insert a <base> tag so relative URLs resolve correctly.
+ */
+export declare const BASE_URL_FIX_SCRIPT: string;
 export declare function generateDumpScriptTag(json: string, attributes?: Record<string, string>): string;
 export {};

package/dist/types/index.d.ts

@@ -2,7 +2,7 @@ import { z } from 'zod';
 import Service from './service/index';
 import { TaskRunner } from './task-runner';
 import { getVersion } from './utils';
-export { plan, AiLocateElement, getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, } from './ai-model/index';
+export { plan, AiLocateElement, getMidsceneLocationSchema, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, } from './ai-model/index';
 export { MIDSCENE_MODEL_NAME, type CreateOpenAIClientFn, } from '@midscene/shared/env';
 export type * from './types';
 export { ServiceError, ExecutionDump, GroupedActionDump, type IExecutionDump, type IGroupedActionDump, } from './types';

package/dist/types/report.d.ts

@@ -3,6 +3,11 @@ export declare class ReportMergingTool {
     private reportInfos;
     append(reportInfo: ReportFileWithAttributes): void;
     clear(): void;
+    /**
+     * Check if a report is in directory mode (html-and-external-assets).
+     * Directory mode reports: {name}/index.html + {name}/screenshots/
+     */
+    private isDirectoryModeReport;
     mergeReports(reportFileName?: 'AUTO' | string, opts?: {
         rmOriginalReports?: boolean;
         overwrite?: boolean;

package/dist/types/types.d.ts

@@ -70,8 +70,9 @@ export interface AgentDescribeElementAtPointResult {
  */
 export declare abstract class UIContext {
     abstract screenshot: ScreenshotItem;
-    abstract size: Size;
+    abstract shotSize: Size;
     abstract _isFrozen?: boolean;
+    abstract deprecatedDpr?: number;
 }
 export type EnsureObject<T> = {
     [K in keyof T]: any;
@@ -492,7 +493,6 @@ export interface WebElementInfo extends BaseElement {
         [key: string]: string;
     };
 }
-export type WebUIContext = UIContext;
 /**
  * Agent
  */

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.4.6",
+  "version": "1.4.7",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "3.24.3",
-    "@midscene/shared": "1.4.6"
+    "@midscene/shared": "1.4.7"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",