@midscene/core 1.12.3 → 1.12.4-beta-20260902093735.0

package/dist/types/agent/agent.d.ts

@@ -1,6 +1,6 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertResult, type AgentOpt, type AgentProgressListener, type AgentWaitForOpt, type AiActEffort, type AssertOptions, type DeepThinkOption, type DeviceAction, ExecutionDump, type InsightAPI, type LocateOption, type LocateResultElement, type OnTaskStartTip, type QueryOptions, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractParam, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type AgentAssertResult, type AgentContextKey, type AgentOpt, type AgentProgressListener, type AgentWaitForOpt, type AiActEffort, type AssertOptions, type DeepThinkOption, type DeviceAction, ExecutionDump, type InsightAPI, type LocateOption, type LocateResultElement, type OnTaskStartTip, type QueryOptions, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractParam, type UIContext } from '../types';
 import type { AbstractInterface, InputStrategy } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
@@ -17,6 +17,11 @@ export type AiActOptions = {
     deepThink?: DeepThinkOption;
     deepLocate?: boolean;
     abortSignal?: AbortSignal;
+    /**
+     * Additional facts, rules, constraints, or output requirements for this AI
+     * call. It overrides `contexts.aiAct` and `contexts.default`; `''` disables
+     * inherited user context for this call.
+     */
     context?: string;
 };
 /**
@@ -63,7 +68,10 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     private activeObserver;
     /** Observers own temporary frame files until their observation is disposed. */
     private ownedObservers;
-    private get aiActContext();
+    private get contexts();
+    private resolveUserContext;
+    private resolveContext;
+    private withContext;
     private executionDumpIndexByRunner;
     private fullActionSpace;
     private activeFileChooserAccepter?;
@@ -123,10 +131,21 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      */
     startObserving(opt?: UIObserverOption): Promise<UIObserver>;
     /**
-     * @deprecated Use {@link setAIActContext} instead.
+     * @deprecated Use `setContext('aiAct', context)` instead.
      */
     setAIActionContext(prompt: string): Promise<void>;
+    /**
+     * @deprecated Use `setContext('aiAct', context)` instead.
+     */
     setAIActContext(prompt: string): Promise<void>;
+    /**
+     * Set Agent-level AI guidance. Use `default` as the shared fallback for all
+     * AI-powered APIs, or an API name to override that fallback for the API.
+     * API-specific values are not implicitly concatenated with `default`.
+     * Passing `undefined` removes an API override; passing `''` keeps an explicit
+     * empty value and therefore prevents that API from using `default`.
+     */
+    setContext(target: AgentContextKey, context: string | undefined): void;
     resetDump(): ReportActionDump;
     appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
     /**

package/dist/types/agent/index.d.ts

@@ -8,6 +8,7 @@ export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
 export type { MidsceneUsageMetrics, UsageBucket } from './metrics';
 export type { GherkinStepKeyword, RunGherkinScenarioOptions, } from './run-gherkin-scenario';
-export type { AgentOpt } from '../types';
+export type { AgentContextKey, AgentContexts, AgentOpt, AiApiName, } from '../types';
 export type { RecordToReportOptions, RecordToReportScreenshot } from '../types';
 export type { AiActOptions } from './agent';
+export { mergeAIContexts } from './prompt-context';

package/dist/types/agent/insight.d.ts

@@ -1,6 +1,8 @@
 import type { TUserPrompt } from '../ai-model/index';
 import type { ModelRuntime } from '../ai-model/models';
 import type { AgentAssertResult, AssertOptions, InsightAPI, QueryOptions, ServiceExtractParam, UIContext } from '../types';
+import type { AiApiName } from '@midscene/shared/agent-tools/agent-context';
+import { type ResolvedAIContext } from './prompt-context';
 import { type TaskExecutor } from './tasks';
 type InsightTaskExecutor = Pick<TaskExecutor, 'createTypeQueryExecution'>;
 /** Execute read-only AI operations against either live or fixed UI context. */
@@ -8,7 +10,10 @@ export declare class Insight implements InsightAPI {
     private readonly taskExecutor;
     private readonly resolveModelRuntime;
     private readonly getUIContext?;
-    constructor(taskExecutor: InsightTaskExecutor, resolveModelRuntime: () => ModelRuntime, getUIContext?: (() => UIContext) | undefined);
+    private readonly resolveAgentContext?;
+    constructor(taskExecutor: InsightTaskExecutor, resolveModelRuntime: () => ModelRuntime, getUIContext?: (() => UIContext) | undefined, resolveAgentContext?: ((apiName: AiApiName, callContext?: string) => ResolvedAIContext | undefined) | undefined);
+    private resolveContext;
+    private withContext;
     private executionOptions;
     aiQuery<ReturnType = any>(demand: ServiceExtractParam, options?: QueryOptions): Promise<ReturnType>;
     aiBoolean(prompt: TUserPrompt, options?: QueryOptions): Promise<boolean>;

package/dist/types/agent/prompt-context.d.ts

@@ -1,3 +1,29 @@
 import type { TUserPrompt } from '../ai-model';
+import type { AiApiName } from '@midscene/shared/agent-tools/agent-context';
+export type AIContextMetadata = {
+    source: 'default';
+} | {
+    source: 'api';
+    apiName: AiApiName;
+} | {
+    source: 'call';
+};
+export interface ResolvedAIContext {
+    value: string;
+    metadata: AIContextMetadata;
+}
+export declare const INTERNAL_AI_CONTEXT_METADATA_KEY: "_internalAIContextMetadata";
+export interface InternalAIContextOptions {
+    [INTERNAL_AI_CONTEXT_METADATA_KEY]?: AIContextMetadata;
+    /** Framework-owned workflow history appended after the selected context. */
+    _internalAdditionalContext?: string;
+}
+export declare const resolvedAIContextFromOptions: (options: ({
+    context?: string;
+} & Partial<InternalAIContextOptions>) | undefined) => ResolvedAIContext | undefined;
+/** Render the selected user context and framework history as separate blocks. */
+export declare const renderAIContext: (resolvedContext: ResolvedAIContext | undefined, workflowHistory?: string) => string | undefined;
+/** Explicitly concatenate context fragments in the order provided. */
+export declare const mergeAIContexts: (...contexts: Array<string | undefined>) => string | undefined;
 export declare const buildPromptWithContext: (prompt: TUserPrompt, context: string | undefined) => TUserPrompt;
-export declare const buildLocatePromptWithContext: (prompt: TUserPrompt, context: string | undefined) => TUserPrompt;
+export declare const buildLocatePromptWithContext: (prompt: TUserPrompt, context: string | undefined, metadata?: AIContextMetadata) => TUserPrompt;

package/dist/types/agent/run-gherkin-scenario.d.ts

@@ -1,6 +1,7 @@
 import type { AiActOptions } from './agent';
 export type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';
 export type RunGherkinScenarioOptions = AiActOptions & {
+    /** Additional AI guidance applied to every step in this scenario run. */
     context?: string;
 };
 type GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';

package/dist/types/index.d.ts

@@ -10,7 +10,7 @@ export { z };
 export default Service;
 export { TaskRunner, Service, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
-export { Agent, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type MidsceneUsageMetrics, type RunGherkinScenarioOptions, type UsageBucket, createAgent, type UIObservation, type UIObserver, type UIObserverOption, } from './agent';
+export { Agent, mergeAIContexts, type AgentOpt, type AiActOptions, type GherkinStepKeyword, type MidsceneUsageMetrics, type RunGherkinScenarioOptions, type UsageBucket, createAgent, type UIObservation, type UIObserver, type UIObserverOption, } from './agent';
 export { describeElementAtPoint, verifyElementDescriptionAtPoint, verifyLocator, type DescribeElementAtPointOptions, type DescribeElementCoordinateSpace, type ElementDescriberRuntime, type VerifyElementDescriptionAtPointOptions, } from './element-describer';
 export { createInlineImageResolver, restoreImageReferences, restoreReportImageReferences, escapeContent, unescapeContent, parseImageScripts, parseDumpScript, parseDumpScriptAttributes, generateImageScriptTag, generateDumpScriptTag, deriveTaskStatus, deriveCaseStatus, } from './dump';
 export type { TaskStatusFields, DerivedTaskStatus, RestoredScreenshotReference, StoredImageReferenceResolver, } from './dump';

package/dist/types/types.d.ts

@@ -1,3 +1,4 @@
+import type { AgentContexts } from '@midscene/shared/agent-tools/agent-context';
 import type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';
 import type { NodeType } from '@midscene/shared/constants';
 import type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';
@@ -8,6 +9,7 @@ import type { ScreenshotItem } from './screenshot-item';
 import type { DetailedLocateParam, MidsceneYamlFlowItem, ServiceExtractOption } from './yaml';
 export type { ElementTreeNode, BaseElement, Rect, Size, Point, } from '@midscene/shared/types';
 export type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';
+export type { AgentContextKey, AgentContexts, AiApiName, } from '@midscene/shared/agent-tools/agent-context';
 export * from './yaml';
 export { ServiceError } from './errors';
 export { ExecutionDump, ReportActionDump, GroupedActionDump, } from './dump/report-action-dump';
@@ -228,6 +230,11 @@ export interface AgentWaitForOpt extends ServiceExtractOption {
 }
 export interface AgentAssertOpt {
     keepRawResponse?: boolean;
+    /**
+     * Additional facts, decision rules, constraints, or output requirements for
+     * this assertion. It overrides `contexts.aiAssert` and `contexts.default`;
+     * `''` disables inherited user context for this call.
+     */
     context?: string;
     abortSignal?: AbortSignal;
 }
@@ -678,7 +685,26 @@ export interface AgentOpt {
      */
     outputFormat?: 'single-html' | 'html-and-external-assets';
     onTaskStartTip?: OnTaskStartTip;
+    /**
+     * Agent-level AI guidance containing business facts, rules, constraints, or
+     * output requirements. `contexts.default` is a shared fallback used only
+     * when neither the call nor the matching API key provides a context.
+     *
+     * A per-call `options.context` overrides the matching API context, which
+     * overrides `contexts.default`. An empty string explicitly clears inherited
+     * user context for that scope. These layers are not implicitly concatenated.
+     */
+    contexts?: AgentContexts;
+    /**
+     * Compatibility alias for `contexts.aiAct`. `contexts.aiAct` takes
+     * precedence when both are provided.
+     * @deprecated Use `contexts.aiAct` instead.
+     */
     aiActContext?: string;
+    /**
+     * Older compatibility alias for `contexts.aiAct`.
+     * @deprecated Use `contexts.aiAct` instead.
+     */
     aiActionContext?: string;
     reportFileName?: string;
     reportAttributes?: ReportAttributes;

package/dist/types/yaml.d.ts

@@ -4,7 +4,11 @@ import type { AgentOpt, LocateResultElement } from './types';
 import type { UIContext } from './types';
 export interface LocateOption extends Partial<TMultimodalPrompt> {
     prompt?: TUserPrompt;
-    /** Additional context for this AI request. */
+    /**
+     * Additional facts, rules, or constraints for this AI call. It overrides
+     * the matching API context and `contexts.default`; `''` disables inherited
+     * user context for this call.
+     */
     context?: string;
     deepLocate?: boolean;
     /** @deprecated Use `deepLocate` instead. Kept for backward compatibility. */
@@ -15,7 +19,11 @@ export interface LocateOption extends Partial<TMultimodalPrompt> {
     fileChooserAccept?: string | string[];
 }
 export interface ServiceExtractOption {
-    /** Additional context for this AI request. */
+    /**
+     * Additional facts, decision rules, constraints, or output requirements for
+     * this AI call. It overrides the matching API context and
+     * `contexts.default`; `''` disables inherited user context for this call.
+     */
     context?: string;
     domIncluded?: boolean | 'visible-only';
     screenshotIncluded?: boolean;
@@ -25,7 +33,7 @@ export interface DetailedLocateParam extends Omit<LocateOption, 'context' | 'dee
     prompt: TUserPrompt;
     /** Original prompt text used for user-facing reports. */
     promptDisplay?: string;
-    /** Per-call business context used for user-facing reports. */
+    /** Per-call AI guidance recorded in user-facing reports. */
     context?: string;
 }
 export type ScrollType = 'singleAction' | 'scrollToBottom' | 'scrollToTop' | 'scrollToRight' | 'scrollToLeft' | 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
@@ -78,7 +86,7 @@ export interface MidsceneYamlTask {
  *     strategy: "read-write"
  * ```
  */
-export type MidsceneYamlScriptAgentOpt = Pick<AgentOpt, 'testId' | 'groupName' | 'groupDescription' | 'generateReport' | 'persistExecutionDump' | 'autoPrintReportMsg' | 'reportFileName' | 'replanningCycleLimit' | 'aiActContext' | 'aiActionContext' | 'cache' | 'screenshotShrinkFactor'>;
+export type MidsceneYamlScriptAgentOpt = Pick<AgentOpt, 'testId' | 'groupName' | 'groupDescription' | 'generateReport' | 'persistExecutionDump' | 'autoPrintReportMsg' | 'reportFileName' | 'replanningCycleLimit' | 'contexts' | 'aiActContext' | 'aiActionContext' | 'cache' | 'screenshotShrinkFactor'>;
 export interface MidsceneYamlScriptConfig {
     output?: string;
     unstableLogContent?: boolean | string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.12.3",
+  "version": "1.12.4-beta-20260902093735.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -153,7 +153,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.12.3"
+    "@midscene/shared": "1.12.4-beta-20260902093735.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",