@midscene/core 0.26.7-beta-20250818025746.0 → 0.26.7-beta-20250818034910.0

package/dist/types/ai-model/common.d.ts

@@ -1,4 +1,4 @@
-import type { AIUsageInfo, BaseElement, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { AIUsageInfo, BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { PlanningLocateParam } from '../types';
 export type AIArgs = [
@@ -12,6 +12,7 @@ export declare enum AIActionType {
     PLAN = 3,
     DESCRIBE_ELEMENT = 4
 }
+export declare const actionSpaceTypePrefix = "action_space_";
 export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
     content: T;
     usage?: AIUsageInfo;
@@ -31,4 +32,4 @@ export declare function mergeRects(rects: Rect[]): {
 };
 export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
 export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
-export declare function buildYamlFlowFromPlans(plans: PlanningAction[], sleep?: number): MidsceneYamlFlowItem[];
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction[], sleep?: number): MidsceneYamlFlowItem[];

package/dist/types/ai-model/index.d.ts

@@ -4,8 +4,9 @@ export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/ut
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
-export { AiLocateElement, AiExtractElementInfo, AiAssert, AiLocateSection, } from './inspect';
+export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
 export { plan } from './llm-planning';
 export { callAiFn, adaptBboxToRect, } from './common';
 export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { AIActionType, type AIArgs } from './common';
+export { actionSpaceTypePrefix } from './common';

package/dist/types/ai-model/inspect.d.ts

@@ -1,4 +1,4 @@
-import type { AIAssertionResponse, AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import { callAiFn } from './common';
 export type AIArgs = [
@@ -40,10 +40,3 @@ export declare function AiExtractElementInfo<T, ElementType extends BaseElement
     elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;
-export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
-    assertion: TUserPrompt;
-    context: UIContext<ElementType>;
-}): Promise<{
-    content: AIAssertionResponse;
-    usage: AIUsageInfo | undefined;
-}>;

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -1,5 +1,2 @@
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
-export declare function systemPromptToAssert(model: {
-    isUITars: boolean;
-}): string;
 export declare const assertSchema: ResponseFormatJSONSchema;

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
 import { Executor } from './ai-model/action-executor';
 import Insight from './insight/index';
 import { getVersion } from './utils';
-export { plan, describeUserPage, AiLocateElement, AiAssert, } from './ai-model/index';
+export { plan, describeUserPage, AiLocateElement, } from './ai-model/index';
 export { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
 export type * from './types';
 export default Insight;

package/dist/types/insight/index.d.ts

@@ -1,5 +1,5 @@
 import { callAiFn } from '../ai-model/common';
-import type { AIDescribeElementResponse, AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, Rect, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIDescribeElementResponse, AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, Rect, TMultimodalPrompt, UIContext } from '../types';
 export interface LocateOpts {
     context?: UIContext<BaseElement>;
     callAI?: typeof callAiFn<AIElementResponse>;
@@ -19,7 +19,6 @@ export default class Insight<ElementType extends BaseElement = BaseElement, Cont
         thought?: string;
         usage?: AIUsageInfo;
     }>;
-    assert(assertion: TUserPrompt): Promise<InsightAssertionResponse>;
     describe(target: Rect | [number, number], opt?: {
         deepThink?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;

package/dist/types/types.d.ts

@@ -390,6 +390,7 @@ export type TUserPrompt = string | ({
 } & Partial<TMultimodalPrompt>);
 export interface DeviceAction<ParamType = any> {
     name: string;
+    interfaceAlias?: string;
     description?: string;
     paramSchema?: string;
     paramDescription?: string;

package/dist/types/yaml.d.ts

@@ -10,6 +10,7 @@ export interface InsightExtractOption {
     domIncluded?: boolean | 'visible-only';
     screenshotIncluded?: boolean;
     returnThought?: boolean;
+    isWaitForAssert?: boolean;
 }
 export interface ReferenceImage {
     base64: string;
@@ -110,16 +111,15 @@ export interface MidsceneYamlFlowItemAIHover extends LocateOption {
     aiHover: TUserPrompt;
 }
 export interface MidsceneYamlFlowItemAIInput extends LocateOption {
-    aiInput: string;
-    locate: TUserPrompt;
+    aiInput: TUserPrompt | undefined;
+    value: string;
 }
 export interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
-    aiKeyboardPress: string;
-    locate?: TUserPrompt;
+    aiKeyboardPress: TUserPrompt | undefined;
+    key: string;
 }
 export interface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {
-    aiScroll: null;
-    locate?: TUserPrompt;
+    aiScroll: TUserPrompt | undefined;
 }
 export interface MidsceneYamlFlowItemEvaluateJavaScript {
     javascript: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.26.7-beta-20250818025746.0",
+  "version": "0.26.7-beta-20250818034910.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -60,8 +60,8 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/recorder": "0.26.7-beta-20250818025746.0",
-    "@midscene/shared": "0.26.7-beta-20250818025746.0"
+    "@midscene/shared": "0.26.7-beta-20250818034910.0",
+    "@midscene/recorder": "0.26.7-beta-20250818034910.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",