@midscene/core 1.1.1-beta-20260107032519.0 → 1.1.1-beta-20260108085624.0

package/dist/types/agent/agent.d.ts

@@ -7,6 +7,7 @@ import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
 export type AiActOptions = {
     cacheable?: boolean;
+    fileChooserAccept?: string | string[];
     deepThink?: DeepThinkOption;
 };
 export declare class Agent<InterfaceType extends AbstractInterface = AbstractInterface> {
@@ -73,7 +74,9 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     private callbackOnTaskStartTip;
     wrapActionInActionSpace<T extends DeviceAction>(name: string): (param: ActionParam<T>) => Promise<ActionReturn<T>>;
     callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
-    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption & {
+        fileChooserAccept?: string | string[];
+    }): Promise<any>;
     aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
     aiDoubleClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
     aiHover(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
@@ -189,6 +192,8 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * Process cache configuration and return normalized cache settings
      */
     private processCacheConfig;
+    private normalizeFilePaths;
+    private normalizeFileInput;
     /**
      * Manually flush cache to file
      * @param options - Optional configuration

package/dist/types/agent/tasks.d.ts

@@ -46,12 +46,14 @@ export declare class TaskExecutor {
         runner: TaskRunner;
     }>;
     runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
-    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption): Promise<ExecutionResult<{
+    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption, fileChooserAccept?: string[]): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
     } | undefined>>;
+    private runAction;
     private createTypeQueryTask;
     createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
     private sleepPlan;
     taskForSleep(timeMs: number, _modelConfig: IModelConfig): Promise<import("../types").ExecutionTaskActionApply<PlanningActionParamSleep>>;
     waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
 }
+export declare function withFileChooser<T>(interfaceInstance: AbstractInterface, fileChooserAccept: string[] | undefined, action: () => Promise<T>): Promise<T>;

package/dist/types/device/index.d.ts

@@ -3,6 +3,9 @@ import type { IModelConfig } from '@midscene/shared/env';
 import type { ElementNode } from '@midscene/shared/extractor';
 import { z } from 'zod';
 import type { ElementCacheFeature, Rect, Size, UIContext } from '../types';
+export interface FileChooserHandler {
+    accept(files: string[]): Promise<void>;
+}
 export declare abstract class AbstractInterface {
     abstract interfaceType: string;
     abstract screenshotBase64(): Promise<string>;
@@ -17,6 +20,10 @@ export declare abstract class AbstractInterface {
     abstract describe?(): string;
     abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;
     abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;
+    registerFileChooserListener?(handler: (chooser: FileChooserHandler) => Promise<void>): Promise<{
+        dispose: () => void;
+        getError: () => Error | undefined;
+    }>;
     abstract getElementsNodeTree?: () => Promise<ElementNode>;
     abstract url?: () => string | Promise<string>;
     abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;

package/dist/types/types.d.ts

@@ -25,12 +25,6 @@ export declare enum AIResponseFormat {
     JSON = "json_object",
     TEXT = "text"
 }
-export type AISingleElementResponseById = {
-    id: string;
-    reason?: string;
-    text?: string;
-    xpaths?: string[];
-};
 export type AISingleElementResponseByPosition = {
     position?: {
         x: number;
@@ -40,7 +34,6 @@ export type AISingleElementResponseByPosition = {
     reason: string;
     text: string;
 };
-export type AISingleElementResponse = AISingleElementResponseById;
 export interface AIElementCoordinatesResponse {
     bbox: [number, number, number, number];
     errors?: string[];

package/dist/types/yaml.d.ts

@@ -8,6 +8,7 @@ export interface LocateOption {
     cacheable?: boolean;
     xpath?: string;
     uiContext?: UIContext;
+    fileChooserAccept?: string | string[];
 }
 export interface ServiceExtractOption {
     domIncluded?: boolean | 'visible-only';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.1.1-beta-20260107032519.0",
+  "version": "1.1.1-beta-20260108085624.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -83,19 +83,19 @@
     "dayjs": "^1.11.11",
     "dotenv": "^16.4.5",
     "fetch-socks": "^1.3.0",
-    "openai": "6.3.0",
-    "undici": "^6.0.0",
+    "js-yaml": "4.1.0",
     "jsonrepair": "3.12.0",
+    "openai": "6.3.0",
     "semver": "7.5.2",
-    "js-yaml": "4.1.0",
+    "undici": "^6.0.0",
     "zod": "3.24.3",
-    "@midscene/shared": "1.1.1-beta-20260107032519.0"
+    "@midscene/shared": "1.1.1-beta-20260108085624.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",
+    "@types/js-yaml": "4.0.9",
     "@types/node": "^18.0.0",
     "@types/node-fetch": "2.6.11",
-    "@types/js-yaml": "4.0.9",
     "@types/semver": "7.7.0",
     "typescript": "^5.8.3",
     "vitest": "3.0.5"