@midscene/core 1.0.1-beta-20251024063839.0 → 1.0.1-beta-20251027033034.0

package/dist/types/agent/index.d.ts

@@ -5,5 +5,5 @@ export { locateParamStr, paramStr, taskTitleStr, typeStr } from './ui-utils';
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
-export { getCurrentExecutionFile, trimContextByViewport, } from './utils';
+export { getCurrentExecutionFile } from './utils';
 export type { AgentOpt } from '../types';

package/dist/types/agent/utils.d.ts

@@ -1,6 +1,6 @@
 import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
 import type { AbstractInterface } from '../device';
-import type { BaseElement, ElementCacheFeature, ElementTreeNode, ExecutionDump, ExecutorContext, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
+import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
 import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
@@ -13,42 +13,11 @@ export declare function printReportMsg(filepath: string): void;
  */
 export declare function getCurrentExecutionFile(trace?: string): string | false;
 export declare function generateCacheId(fileName?: string): string;
-export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
+export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam): LocateResultElement | undefined;
 export declare function matchElementFromCache(context: {
     taskCache?: TaskCache;
     interfaceInstance: AbstractInterface;
 }, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
-export declare function trimContextByViewport(execution: ExecutionDump): {
-    tasks: {
-        type: any;
-        subType?: string;
-        subTask?: boolean;
-        param?: any;
-        thought?: string;
-        locate?: PlanningLocateParam | null;
-        uiContext?: UIContext;
-        executor: (param: any, context: ExecutorContext) => void | Promise<void | import("../types").ExecutionTaskReturn<any, any> | undefined> | undefined;
-        output?: any;
-        log?: any;
-        recorder?: import("../types").ExecutionRecorderItem[];
-        hitBy?: import("../types").ExecutionTaskHitBy;
-        status: "pending" | "running" | "finished" | "failed" | "cancelled";
-        error?: Error;
-        errorMessage?: string;
-        errorStack?: string;
-        timing?: {
-            start: number;
-            end?: number;
-            cost?: number;
-        };
-        usage?: import("../types").AIUsageInfo;
-        searchAreaUsage?: import("../types").AIUsageInfo;
-    }[];
-    name: string;
-    description?: string;
-    aiActionContext?: string;
-    logTime: number;
-};
 export declare const getMidsceneVersion: () => string;
 export declare const parsePrompt: (prompt: TUserPrompt) => {
     textPrompt: string;

package/dist/types/ai-model/index.d.ts

@@ -1,6 +1,6 @@
 export { callAIWithStringResponse, callAIWithObjectResponse, callAI, } from './service-caller/index';
 export { systemPromptToLocateElement } from './prompt/llm-locator';
-export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
+export { describeUserPage } from './prompt/util';
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';

package/dist/types/ai-model/inspect.d.ts

@@ -1,5 +1,6 @@
-import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementResponse, AIUsageInfo, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
+import type { LocateResultElement } from '@midscene/shared/types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { TMultimodalPrompt, TUserPrompt } from './common';
 import { callAIWithObjectResponse } from './service-caller/index';
@@ -7,23 +8,24 @@ export type AIArgs = [
     ChatCompletionSystemMessageParam,
     ...ChatCompletionUserMessageParam[]
 ];
-export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
-    context: UIContext<ElementType>;
+export declare function AiLocateElement(options: {
+    context: UIContext;
     targetElementDescription: TUserPrompt;
     referenceImage?: ReferenceImage;
     callAIFn: typeof callAIWithObjectResponse<AIElementResponse | [number, number]>;
     searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
     modelConfig: IModelConfig;
 }): Promise<{
-    parseResult: AIElementLocatorResponse;
+    parseResult: {
+        elements: LocateResultElement[];
+        errors?: string[];
+    };
     rect?: Rect;
     rawResponse: string;
-    elementById: ElementById;
     usage?: AIUsageInfo;
-    isOrderSensitive?: boolean;
 }>;
 export declare function AiLocateSection(options: {
-    context: UIContext<BaseElement>;
+    context: UIContext;
     sectionDescription: TUserPrompt;
     modelConfig: IModelConfig;
 }): Promise<{
@@ -33,14 +35,14 @@ export declare function AiLocateSection(options: {
     rawResponse: string;
     usage?: AIUsageInfo;
 }>;
-export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+export declare function AiExtractElementInfo<T>(options: {
     dataQuery: string | Record<string, string>;
     multimodalPrompt?: TMultimodalPrompt;
-    context: UIContext<ElementType>;
+    context: UIContext;
+    pageDescription?: string;
     extractOption?: InsightExtractOption;
     modelConfig: IModelConfig;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
-    elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,17 +1,9 @@
-import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { BaseElement, Size, UIContext } from '../../types';
 export declare function describeSize(size: Size): string;
 export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
     id: string;
 })[]): string;
 export declare const distanceThreshold = 16;
-export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
-    x: number;
-    y: number;
-}, options?: {
-    requireStrictDistance?: boolean;
-    filterPositionElements?: boolean;
-}): BaseElement | undefined;
 export declare function distance(point1: {
     x: number;
     y: number;
@@ -20,28 +12,4 @@ export declare function distance(point1: {
     y: number;
 }): number;
 export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
-export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt: {
-    truncateTextLength?: number;
-    filterNonTextContent?: boolean;
-    domIncluded?: boolean | 'visible-only';
-    visibleOnly?: boolean;
-    vlMode: TVlModeTypes | undefined;
-}): Promise<{
-    description: string;
-    elementById(idOrIndexId: string): ElementType;
-    elementByPosition(position: {
-        x: number;
-        y: number;
-    }, size: {
-        width: number;
-        height: number;
-    }): BaseElement | undefined;
-    insertElementByPosition(position: {
-        x: number;
-        y: number;
-    }): ElementType;
-    size: {
-        width: number;
-        height: number;
-    };
-}>;
+export declare function describeUserPage(context: UIContext): Promise<string>;

package/dist/types/insight/index.d.ts

@@ -1,9 +1,9 @@
 import { callAIWithObjectResponse } from '../ai-model/index';
-import type { AIDescribeElementResponse, BaseElement, DetailedLocateParam, InsightExtractOption, InsightExtractParam, InsightExtractResult, InsightTaskInfo, LocateResultWithDump, Rect, UIContext } from '../types';
+import type { AIDescribeElementResponse, DetailedLocateParam, InsightExtractOption, InsightExtractParam, InsightExtractResult, InsightTaskInfo, LocateResultWithDump, Rect, UIContext } from '../types';
 import { type IModelConfig } from '@midscene/shared/env';
 import type { TMultimodalPrompt } from '../ai-model/common';
 export interface LocateOpts {
-    context?: UIContext<BaseElement>;
+    context?: UIContext;
 }
 export type AnyValue<T> = {
     [K in keyof T]: unknown extends T[K] ? any : T[K];
@@ -12,13 +12,13 @@ interface InsightOptions {
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
     aiVendorFn?: typeof callAIWithObjectResponse;
 }
-export default class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
-    contextRetrieverFn: () => Promise<ContextType> | ContextType;
+export default class Insight {
+    contextRetrieverFn: () => Promise<UIContext> | UIContext;
     aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined>;
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
-    constructor(context: ContextType | (() => Promise<ContextType> | ContextType), opt?: InsightOptions);
+    constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: InsightOptions);
     locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResultWithDump>;
-    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<InsightExtractResult<T>>;
+    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt): Promise<InsightExtractResult<T>>;
     describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
         deepThink?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;

package/dist/types/tree.d.ts

@@ -1 +1,4 @@
-export { truncateText, trimAttributes, descriptionOfTree, } from '@midscene/shared/extractor';
+import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';
+import { trimAttributes, truncateText } from '@midscene/shared/extractor';
+export { trimAttributes, truncateText };
+export declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean, visibleOnly?: boolean): string;

package/dist/types/types.d.ts

@@ -1,6 +1,6 @@
 import type { NodeType } from '@midscene/shared/constants';
 import type { CreateOpenAIClientFn, TModelConfigFn } from '@midscene/shared/env';
-import type { BaseElement, ElementTreeNode, Rect, Size } from '@midscene/shared/types';
+import type { BaseElement, LocateResultElement, Rect, Size } from '@midscene/shared/types';
 import type { z } from 'zod';
 import type { TUserPrompt } from './ai-model/common';
 import type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';
@@ -15,6 +15,7 @@ export type AIUsageInfo = Record<string, any> & {
     model_description: string | undefined;
     intent: string | undefined;
 };
+export type { LocateResultElement };
 /**
  * openai
  *
@@ -39,23 +40,12 @@ export type AISingleElementResponseByPosition = {
     text: string;
 };
 export type AISingleElementResponse = AISingleElementResponseById;
-export interface AIElementLocatorResponse {
-    elements: {
-        id: string;
-        reason?: string;
-        text?: string;
-        xpaths?: string[];
-    }[];
-    bbox?: [number, number, number, number];
-    isOrderSensitive?: boolean;
-    errors?: string[];
-}
 export interface AIElementCoordinatesResponse {
     bbox: [number, number, number, number];
     isOrderSensitive?: boolean;
     errors?: string[];
 }
-export type AIElementResponse = AIElementLocatorResponse | AIElementCoordinatesResponse;
+export type AIElementResponse = AIElementCoordinatesResponse;
 export interface AIDataExtractionResponse<DataDemand> {
     data: DataDemand;
     errors?: string[];
@@ -91,9 +81,8 @@ export interface AgentDescribeElementAtPointResult {
 /**
  * context
  */
-export declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
+export declare abstract class UIContext {
     abstract screenshotBase64: string;
-    abstract tree: ElementTreeNode<ElementType>;
     abstract size: Size;
     abstract _isFrozen?: boolean;
 }
@@ -103,18 +92,6 @@ export type EnsureObject<T> = {
 export type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';
 export type InsightExtractParam = string | Record<string, string>;
 export type ElementCacheFeature = Record<string, unknown>;
-export type LocateResultElement = {
-    center: [number, number];
-    rect: Rect;
-    id: string;
-    indexId?: number;
-    xpaths: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    isOrderSensitive?: boolean;
-};
 export interface LocateResult {
     element: LocateResultElement | null;
     rect?: Rect;
@@ -143,7 +120,7 @@ export interface InsightDump extends DumpMeta {
         dataDemand?: InsightExtractParam;
         assertion?: TUserPrompt;
     };
-    matchedElement: BaseElement[];
+    matchedElement: LocateResultElement[];
     matchedRect?: Rect;
     deepThink?: boolean;
     data: any;
@@ -193,7 +170,6 @@ export interface AgentAssertOpt {
  *
  */
 export interface PlanningLocateParam extends DetailedLocateParam {
-    id?: string;
     bbox?: [number, number, number, number];
 }
 export interface PlanningAction<ParamType = any> {
@@ -385,7 +361,7 @@ export interface WebElementInfo extends BaseElement {
         [key: string]: string;
     };
 }
-export type WebUIContext = UIContext<WebElementInfo>;
+export type WebUIContext = UIContext;
 /**
  * Agent
  */
@@ -421,14 +397,9 @@ export interface AgentOpt {
      *
      * @example
      * ```typescript
-     * createOpenAIClient: (config) => {
-     *   const openai = new OpenAI({
-     *     apiKey: config.openaiApiKey,
-     *     baseURL: config.openaiBaseURL,
-     *   });
-     *
+     * createOpenAIClient: async (openai, opts) => {
      *   // Wrap with langsmith for planning tasks
-     *   if (config.intent === 'planning') {
+     *   if (opts.baseURL?.includes('planning')) {
      *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });
      *   }
      *

package/dist/types/yaml.d.ts

@@ -1,13 +1,13 @@
 import type { TUserPrompt } from './ai-model/common';
 import type { AndroidDeviceOpt, IOSDeviceOpt } from './device';
 import type { AgentOpt, Rect } from './types';
-import type { BaseElement, UIContext } from './types';
+import type { UIContext } from './types';
 export interface LocateOption {
     prompt?: TUserPrompt;
     deepThink?: boolean;
     cacheable?: boolean;
     xpath?: string;
-    uiContext?: UIContext<BaseElement>;
+    uiContext?: UIContext;
 }
 export interface InsightExtractOption {
     domIncluded?: boolean | 'visible-only';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.0.1-beta-20251024063839.0",
+  "version": "1.0.1-beta-20251027033034.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,8 +89,8 @@
     "zod": "3.24.3",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "@midscene/recorder": "1.0.1-beta-20251024063839.0",
-    "@midscene/shared": "1.0.1-beta-20251024063839.0"
+    "@midscene/recorder": "1.0.1-beta-20251027033034.0",
+    "@midscene/shared": "1.0.1-beta-20251027033034.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.11.2",