@midscene/core 0.26.7-beta-20250818035341.0 → 0.26.7-beta-20250818081955.0

package/dist/types/ai-model/common.d.ts

@@ -1,4 +1,4 @@
-import type { AIUsageInfo, BaseElement, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { AIUsageInfo, BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { PlanningLocateParam } from '../types';
 export type AIArgs = [
@@ -12,6 +12,7 @@ export declare enum AIActionType {
     PLAN = 3,
     DESCRIBE_ELEMENT = 4
 }
+export declare const actionSpaceTypePrefix = "action_space_";
 export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
     content: T;
     usage?: AIUsageInfo;
@@ -31,4 +32,4 @@ export declare function mergeRects(rects: Rect[]): {
 };
 export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
 export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
-export declare function buildYamlFlowFromPlans(plans: PlanningAction[], sleep?: number): MidsceneYamlFlowItem[];
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction[], sleep?: number): MidsceneYamlFlowItem[];

package/dist/types/ai-model/index.d.ts

@@ -1,11 +1,12 @@
-export { callAiFnWithStringResponse, callToGetJSONObject, call as callAi, } from './service-caller/index';
+export { callAiFnWithStringResponse, callToGetJSONObject, call as callAi, getModelName, } from './service-caller/index';
 export { systemPromptToLocateElement } from './prompt/llm-locator';
 export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
-export { AiLocateElement, AiExtractElementInfo, AiAssert, AiLocateSection, } from './inspect';
+export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
 export { plan } from './llm-planning';
 export { callAiFn, adaptBboxToRect, } from './common';
 export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { AIActionType, type AIArgs } from './common';
+export { actionSpaceTypePrefix } from './common';

package/dist/types/ai-model/inspect.d.ts

@@ -1,4 +1,4 @@
-import type { AIAssertionResponse, AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import { callAiFn } from './common';
 export type AIArgs = [
@@ -40,10 +40,3 @@ export declare function AiExtractElementInfo<T, ElementType extends BaseElement
     elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;
-export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
-    assertion: TUserPrompt;
-    context: UIContext<ElementType>;
-}): Promise<{
-    content: AIAssertionResponse;
-    usage: AIUsageInfo | undefined;
-}>;

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -1,5 +1,2 @@
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
-export declare function systemPromptToAssert(model: {
-    isUITars: boolean;
-}): string;
 export declare const assertSchema: ResponseFormatJSONSchema;

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
 import { Executor } from './ai-model/action-executor';
 import Insight from './insight/index';
 import { getVersion } from './utils';
-export { plan, describeUserPage, AiLocateElement, AiAssert, } from './ai-model/index';
+export { plan, describeUserPage, AiLocateElement, } from './ai-model/index';
 export { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
 export type * from './types';
 export default Insight;

package/dist/types/insight/index.d.ts

@@ -1,5 +1,5 @@
 import { callAiFn } from '../ai-model/common';
-import type { AIDescribeElementResponse, AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, Rect, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIDescribeElementResponse, AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, Rect, TMultimodalPrompt, UIContext } from '../types';
 export interface LocateOpts {
     context?: UIContext<BaseElement>;
     callAI?: typeof callAiFn<AIElementResponse>;
@@ -19,7 +19,6 @@ export default class Insight<ElementType extends BaseElement = BaseElement, Cont
         thought?: string;
         usage?: AIUsageInfo;
     }>;
-    assert(assertion: TUserPrompt): Promise<InsightAssertionResponse>;
     describe(target: Rect | [number, number], opt?: {
         deepThink?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;

package/dist/types/types.d.ts

@@ -9,6 +9,7 @@ export type AIUsageInfo = Record<string, any> & {
     completion_tokens: number | undefined;
     total_tokens: number | undefined;
     time_cost: number | undefined;
+    model_name: string | undefined;
 };
 /**
  * openai
@@ -132,8 +133,6 @@ export interface InsightTaskInfo {
 export interface DumpMeta {
     sdkVersion: string;
     logTime: number;
-    model_name: string;
-    model_description?: string;
 }
 export interface ReportDumpWithAttributes {
     dumpString: string;
@@ -334,6 +333,8 @@ export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
 export interface GroupedActionDump {
     groupName: string;
     groupDescription?: string;
+    modelName: string;
+    modelDescription: string;
     executions: ExecutionDump[];
 }
 export type PageType = 'puppeteer' | 'playwright' | 'static' | 'chrome-extension-proxy' | 'android';
@@ -390,6 +391,7 @@ export type TUserPrompt = string | ({
 } & Partial<TMultimodalPrompt>);
 export interface DeviceAction<ParamType = any> {
     name: string;
+    interfaceAlias?: string;
     description?: string;
     paramSchema?: string;
     paramDescription?: string;

package/dist/types/yaml.d.ts

@@ -10,6 +10,7 @@ export interface InsightExtractOption {
     domIncluded?: boolean | 'visible-only';
     screenshotIncluded?: boolean;
     returnThought?: boolean;
+    isWaitForAssert?: boolean;
 }
 export interface ReferenceImage {
     base64: string;
@@ -110,16 +111,15 @@ export interface MidsceneYamlFlowItemAIHover extends LocateOption {
     aiHover: TUserPrompt;
 }
 export interface MidsceneYamlFlowItemAIInput extends LocateOption {
-    aiInput: string;
-    locate: TUserPrompt;
+    aiInput: TUserPrompt | undefined;
+    value: string;
 }
 export interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
-    aiKeyboardPress: string;
-    locate?: TUserPrompt;
+    aiKeyboardPress: TUserPrompt | undefined;
+    key: string;
 }
 export interface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {
-    aiScroll: null;
-    locate?: TUserPrompt;
+    aiScroll: TUserPrompt | undefined;
 }
 export interface MidsceneYamlFlowItemEvaluateJavaScript {
     javascript: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.26.7-beta-20250818035341.0",
+  "version": "0.26.7-beta-20250818081955.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -60,8 +60,8 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/shared": "0.26.7-beta-20250818035341.0",
-    "@midscene/recorder": "0.26.7-beta-20250818035341.0"
+    "@midscene/recorder": "0.26.7-beta-20250818081955.0",
+    "@midscene/shared": "0.26.7-beta-20250818081955.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",