@midscene/core 1.12.4-beta-20260904041024.0 → 1.12.4

package/dist/es/agent/agent.mjs

@@ -10,6 +10,7 @@ import { getVersion, processCacheConfig, reportHTMLContent } from "../utils.mjs"
 import { ScriptPlayer, buildDetailedLocateParam, buildDetailedLocateParamAndRestParams, parseYamlScript } from "../yaml/index.mjs";
 import { readFile } from "node:fs/promises";
 import { basename, resolve as external_node_path_resolve } from "node:path";
+import { isAgentAIContextKey } from "@midscene/shared/agent-tools/agent-context";
 import { serializeError } from "@midscene/shared/agent-tools/error-formatter";
 import { observationArtifactAdapterSymbol } from "@midscene/shared/agent-tools/observation-artifact";
 import { MIDSCENE_REPLANNING_CYCLE_LIMIT, ModelConfigManager, globalConfigManager, globalModelConfigManager } from "@midscene/shared/env";
@@ -27,6 +28,7 @@ import { runGherkinScenario } from "./run-gherkin-scenario.mjs";
 import { markdownToAiActPrompt } from "./run-markdown.mjs";
 import { TaskCache } from "./task-cache.mjs";
 import { TaskExecutor, locatePlanForLocate, withFileChooser } from "./tasks.mjs";
+import { commonAgentTestRunnerNodeDefinitions } from "./test-runner-nodes.mjs";
 import { UIObservationImpl, UIObserverImpl, uiContextFromObservationRecord } from "./ui-observer.mjs";
 import { locateParamStr, paramStr, taskTitleStr, typeStr } from "./ui-utils.mjs";
 import { commonContextParser, getReportFileName, normalizeFilePaths, normalizeScrollType } from "./utils.mjs";
@@ -45,6 +47,9 @@ const warn = getDebug('agent', {
     console: true
 });
 class Agent {
+    static getTestRunnerNodeDefinitions() {
+        return commonAgentTestRunnerNodeDefinitions;
+    }
     get onDumpUpdate() {
         return this.dumpUpdateListeners[0];
     }
@@ -52,8 +57,24 @@ class Agent {
         this.dumpUpdateListeners = [];
         if (callback) this.dumpUpdateListeners.push(callback);
     }
-    get aiActContext() {
-        return this.opts.aiActContext ?? this.opts.aiActionContext;
+    get aiContexts() {
+        if (!this.opts.aiContexts) this.opts.aiContexts = {};
+        return this.opts.aiContexts;
+    }
+    resolveUserContext(apiName, callContext) {
+        if (void 0 !== callContext) return callContext;
+        const apiContext = this.opts.aiContexts?.[apiName];
+        if (void 0 !== apiContext) return apiContext;
+        const defaultContext = this.opts.aiContexts?.default;
+        if (void 0 !== defaultContext) return defaultContext;
+    }
+    withContext(apiName, options) {
+        const resolvedContext = this.resolveUserContext(apiName, options?.context);
+        if (void 0 === resolvedContext) return options;
+        return {
+            ...options ?? {},
+            context: resolvedContext
+        };
     }
     get page() {
         return this.interface;
@@ -79,7 +100,7 @@ class Agent {
         };
     }
     createInsight(getUIContext) {
-        return new Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), getUIContext);
+        return new Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), (apiName, callContext)=>this.resolveUserContext(apiName, callContext), getUIContext);
     }
     async getActionSpace() {
         return this.fullActionSpace;
@@ -140,12 +161,24 @@ class Agent {
         return observer;
     }
     async setAIActionContext(prompt) {
-        await this.setAIActContext(prompt);
+        warn('setAIActionContext() is deprecated; use setAIContext("aiAct", context) instead.');
+        this.setAIContext('aiAct', prompt);
     }
     async setAIActContext(prompt) {
-        if (this.aiActContext) console.warn('aiActContext is already set, and it is called again, will override the previous setting');
-        this.opts.aiActContext = prompt;
-        this.opts.aiActionContext = prompt;
+        warn('setAIActContext() is deprecated; use setAIContext("aiAct", context) instead.');
+        this.setAIContext('aiAct', prompt);
+    }
+    setAIContext(target, context) {
+        if (!isAgentAIContextKey(target)) throw new TypeError(`Unknown Agent context key: ${String(target)}`);
+        if (void 0 !== context && 'string' != typeof context) throw new TypeError('Agent context must be a string or undefined');
+        this.aiContexts[target] = context;
+        if ('aiAct' === target) if (void 0 === context) {
+            this.opts.aiActContext = void 0;
+            this.opts.aiActionContext = void 0;
+        } else {
+            this.opts.aiActContext = context;
+            this.opts.aiActionContext = context;
+        }
     }
     resetDump() {
         this.dump = new ReportActionDump({
@@ -249,7 +282,7 @@ class Agent {
     }
     async aiTap(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for tap');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiTap', opt));
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
         await withFileChooser(this.interface, fileChooserAccept, async ()=>{
             await this.callActionInActionSpace('Tap', {
@@ -259,21 +292,21 @@ class Agent {
     }
     async aiRightClick(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for right click');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiRightClick', opt));
         await this.callActionInActionSpace('RightClick', {
             locate: detailedLocateParam
         });
     }
     async aiDoubleClick(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for double click');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiDoubleClick', opt));
         await this.callActionInActionSpace('DoubleClick', {
             locate: detailedLocateParam
         });
     }
     async aiHover(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for hover');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiHover', opt));
         await this.callActionInActionSpace('Hover', {
             locate: detailedLocateParam
         });
@@ -297,7 +330,7 @@ class Agent {
         }
         assert('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
         assert(locatePrompt, 'missing locate prompt for input');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, this.withContext('aiInput', opt));
         const stringValue = 'number' == typeof value ? String(value) : value;
         const mode = opt?.mode === 'append' ? 'typeOnly' : opt?.mode;
         await this.callActionInActionSpace('Input', {
@@ -323,7 +356,7 @@ class Agent {
             };
         }
         assert(opt?.keyName, 'missing keyName for keyboard press');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiKeyboardPress', opt));
         await this.callActionInActionSpace('KeyboardPress', {
             ...restParams,
             locate: locateParam
@@ -355,14 +388,14 @@ class Agent {
                 scrollType: normalizedScrollType
             };
         }
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiScroll', opt));
         await this.callActionInActionSpace('Scroll', {
             ...restParams,
             locate: locateParam
         });
     }
     async aiPinch(locatePrompt, opt) {
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiPinch', opt));
         await this.callActionInActionSpace('Pinch', {
             ...restParams,
             locate: locateParam
@@ -370,7 +403,7 @@ class Agent {
     }
     async aiLongPress(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for long press');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, this.withContext('aiLongPress', opt));
         await this.callActionInActionSpace('LongPress', {
             ...restParams,
             locate: locateParam
@@ -378,13 +411,14 @@ class Agent {
     }
     async aiClearInput(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for clear input');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiClearInput', opt));
         await this.callActionInActionSpace('ClearInput', {
             locate: detailedLocateParam
         });
     }
     async aiAct(taskPrompt, opt) {
-        const internalReportDisplay = opt?._internalReportDisplay;
+        const internalOptions = opt;
+        const internalReportDisplay = internalOptions?._internalReportDisplay;
         const taskPromptText = 'string' == typeof taskPrompt ? taskPrompt : taskPrompt.prompt;
         const reportPrompt = internalReportDisplay?.prompt || taskPromptText;
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
@@ -393,7 +427,7 @@ class Agent {
         const runAiAct = async ()=>{
             const planningModel = this.resolveModelRuntime('planning');
             const defaultModel = this.resolveModelRuntime('default');
-            const aiActContext = opt?.context !== void 0 ? opt.context : this.aiActContext;
+            const aiActContext = this.resolveUserContext('aiAct', opt?.context);
             const cachePrompt = buildPromptWithContext(taskPrompt, aiActContext);
             const effort = (()=>{
                 const resolvedEffort = opt?.effort ?? (opt?.deepThink === true ? 'deepThink' : 'balance');
@@ -511,7 +545,7 @@ class Agent {
         return this.createInsight().aiAsk(prompt, opt);
     }
     async aiLocate(prompt, opt) {
-        const locateParam = buildDetailedLocateParam(prompt, opt);
+        const locateParam = buildDetailedLocateParam(prompt, this.withContext('aiLocate', opt));
         assert(locateParam, 'cannot get locate param for aiLocate');
         const locatePlan = locatePlanForLocate(locateParam);
         const plans = [
@@ -534,10 +568,11 @@ class Agent {
     }
     async aiWaitFor(assertion, opt) {
         const modelRuntime = this.resolveModelRuntime('insight');
+        const options = this.withContext('aiWaitFor', opt);
         await this.taskExecutor.waitFor(assertion, {
-            ...opt,
-            timeoutMs: opt?.timeoutMs || 15000,
-            checkIntervalMs: opt?.checkIntervalMs || 3000
+            ...options,
+            timeoutMs: options?.timeoutMs || 15000,
+            checkIntervalMs: options?.checkIntervalMs || 3000
         }, modelRuntime);
     }
     async ai(...args) {
@@ -806,11 +841,23 @@ class Agent {
             groupDescription: ''
         }, opts || {});
         assertReportGenerationOptions(this.opts);
-        const resolvedAiActContext = this.opts.aiActContext ?? this.opts.aiActionContext;
+        if (void 0 !== this.opts.aiContexts && ('object' != typeof this.opts.aiContexts || null === this.opts.aiContexts || Array.isArray(this.opts.aiContexts))) throw new TypeError('opts.aiContexts must be a plain object');
+        for (const [key, value] of Object.entries(this.opts.aiContexts ?? {})){
+            if (!isAgentAIContextKey(key)) throw new TypeError(`Unknown Agent context key: ${key}`);
+            if (void 0 !== value && 'string' != typeof value) throw new TypeError(`Agent context "${key}" must be a string`);
+        }
+        const deprecatedAiActContextOption = void 0 !== this.opts.aiActContext ? 'aiActContext' : void 0 !== this.opts.aiActionContext ? 'aiActionContext' : void 0;
+        if (deprecatedAiActContextOption) warn(`Agent option "${deprecatedAiActContextOption}" is deprecated; use "aiContexts.aiAct" instead. When both are provided, "aiContexts.aiAct" takes precedence.`);
+        const normalizedAIContexts = {
+            ...this.opts.aiContexts
+        };
+        const resolvedAiActContext = normalizedAIContexts.aiAct ?? this.opts.aiActContext ?? this.opts.aiActionContext;
         if (void 0 !== resolvedAiActContext) {
+            normalizedAIContexts.aiAct = resolvedAiActContext;
             this.opts.aiActContext = resolvedAiActContext;
-            this.opts.aiActionContext ??= resolvedAiActContext;
+            this.opts.aiActionContext = resolvedAiActContext;
         }
+        this.opts.aiContexts = normalizedAIContexts;
         if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
         const hasCustomConfig = opts?.modelConfig || opts?.createOpenAIClient;
         this.modelConfigManager = hasCustomConfig ? new ModelConfigManager(opts?.modelConfig, opts?.createOpenAIClient) : globalModelConfigManager;