@midscene/core 1.12.3 → 1.12.4-beta-20260902093735.0

package/dist/lib/agent/agent.js

@@ -50,6 +50,7 @@ const external_utils_js_namespaceObject = require("../utils.js");
 const external_yaml_index_js_namespaceObject = require("../yaml/index.js");
 const promises_namespaceObject = require("node:fs/promises");
 const external_node_path_namespaceObject = require("node:path");
+const agent_context_namespaceObject = require("@midscene/shared/agent-tools/agent-context");
 const error_formatter_namespaceObject = require("@midscene/shared/agent-tools/error-formatter");
 const observation_artifact_namespaceObject = require("@midscene/shared/agent-tools/observation-artifact");
 const env_namespaceObject = require("@midscene/shared/env");
@@ -92,8 +93,44 @@ class Agent {
         this.dumpUpdateListeners = [];
         if (callback) this.dumpUpdateListeners.push(callback);
     }
-    get aiActContext() {
-        return this.opts.aiActContext ?? this.opts.aiActionContext;
+    get contexts() {
+        if (!this.opts.contexts) this.opts.contexts = {};
+        return this.opts.contexts;
+    }
+    resolveUserContext(apiName, callContext) {
+        if (void 0 !== callContext) return {
+            value: callContext,
+            metadata: {
+                source: 'call'
+            }
+        };
+        const apiContext = this.opts.contexts?.[apiName];
+        if (void 0 !== apiContext) return {
+            value: apiContext,
+            metadata: {
+                source: 'api',
+                apiName
+            }
+        };
+        const defaultContext = this.opts.contexts?.default;
+        if (void 0 !== defaultContext) return {
+            value: defaultContext,
+            metadata: {
+                source: 'default'
+            }
+        };
+    }
+    resolveContext(apiName, callContext, additionalContext) {
+        return (0, external_prompt_context_js_namespaceObject.renderAIContext)(this.resolveUserContext(apiName, callContext), additionalContext);
+    }
+    withContext(apiName, options) {
+        const resolvedContext = this.resolveUserContext(apiName, options?.context);
+        if (void 0 === resolvedContext) return options;
+        return {
+            ...options ?? {},
+            context: resolvedContext.value,
+            [external_prompt_context_js_namespaceObject.INTERNAL_AI_CONTEXT_METADATA_KEY]: resolvedContext.metadata
+        };
     }
     get page() {
         return this.interface;
@@ -119,7 +156,7 @@ class Agent {
         };
     }
     createInsight(getUIContext) {
-        return new external_insight_js_namespaceObject.Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), getUIContext);
+        return new external_insight_js_namespaceObject.Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), getUIContext, (apiName, callContext)=>this.resolveUserContext(apiName, callContext));
     }
     async getActionSpace() {
         return this.fullActionSpace;
@@ -180,12 +217,24 @@ class Agent {
         return observer;
     }
     async setAIActionContext(prompt) {
-        await this.setAIActContext(prompt);
+        warn('setAIActionContext() is deprecated; use setContext("aiAct", context) instead.');
+        this.setContext('aiAct', prompt);
     }
     async setAIActContext(prompt) {
-        if (this.aiActContext) console.warn('aiActContext is already set, and it is called again, will override the previous setting');
-        this.opts.aiActContext = prompt;
-        this.opts.aiActionContext = prompt;
+        warn('setAIActContext() is deprecated; use setContext("aiAct", context) instead.');
+        this.setContext('aiAct', prompt);
+    }
+    setContext(target, context) {
+        if (!(0, agent_context_namespaceObject.isAgentContextKey)(target)) throw new TypeError(`Unknown Agent context key: ${String(target)}`);
+        if (void 0 !== context && 'string' != typeof context) throw new TypeError('Agent context must be a string or undefined');
+        this.contexts[target] = context;
+        if ('aiAct' === target) if (void 0 === context) {
+            this.opts.aiActContext = void 0;
+            this.opts.aiActionContext = void 0;
+        } else {
+            this.opts.aiActContext = context;
+            this.opts.aiActionContext = context;
+        }
     }
     resetDump() {
         this.dump = new external_types_js_namespaceObject.ReportActionDump({
@@ -289,7 +338,7 @@ class Agent {
     }
     async aiTap(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for tap');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, this.withContext('aiTap', opt));
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
         await (0, external_tasks_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>{
             await this.callActionInActionSpace('Tap', {
@@ -299,21 +348,21 @@ class Agent {
     }
     async aiRightClick(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for right click');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, this.withContext('aiRightClick', opt));
         await this.callActionInActionSpace('RightClick', {
             locate: detailedLocateParam
         });
     }
     async aiDoubleClick(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for double click');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, this.withContext('aiDoubleClick', opt));
         await this.callActionInActionSpace('DoubleClick', {
             locate: detailedLocateParam
         });
     }
     async aiHover(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for hover');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, this.withContext('aiHover', opt));
         await this.callActionInActionSpace('Hover', {
             locate: detailedLocateParam
         });
@@ -337,7 +386,7 @@ class Agent {
         }
         (0, utils_namespaceObject.assert)('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
-        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, this.withContext('aiInput', opt));
         const stringValue = 'number' == typeof value ? String(value) : value;
         const mode = opt?.mode === 'append' ? 'typeOnly' : opt?.mode;
         await this.callActionInActionSpace('Input', {
@@ -363,7 +412,7 @@ class Agent {
             };
         }
         (0, utils_namespaceObject.assert)(opt?.keyName, 'missing keyName for keyboard press');
-        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', this.withContext('aiKeyboardPress', opt));
         await this.callActionInActionSpace('KeyboardPress', {
             ...restParams,
             locate: locateParam
@@ -395,14 +444,14 @@ class Agent {
                 scrollType: normalizedScrollType
             };
         }
-        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', this.withContext('aiScroll', opt));
         await this.callActionInActionSpace('Scroll', {
             ...restParams,
             locate: locateParam
         });
     }
     async aiPinch(locatePrompt, opt) {
-        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', this.withContext('aiPinch', opt));
         await this.callActionInActionSpace('Pinch', {
             ...restParams,
             locate: locateParam
@@ -410,7 +459,7 @@ class Agent {
     }
     async aiLongPress(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for long press');
-        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, this.withContext('aiLongPress', opt));
         await this.callActionInActionSpace('LongPress', {
             ...restParams,
             locate: locateParam
@@ -418,13 +467,14 @@ class Agent {
     }
     async aiClearInput(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for clear input');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, this.withContext('aiClearInput', opt));
         await this.callActionInActionSpace('ClearInput', {
             locate: detailedLocateParam
         });
     }
     async aiAct(taskPrompt, opt) {
-        const internalReportDisplay = opt?._internalReportDisplay;
+        const internalOptions = opt;
+        const internalReportDisplay = internalOptions?._internalReportDisplay;
         const taskPromptText = 'string' == typeof taskPrompt ? taskPrompt : taskPrompt.prompt;
         const reportPrompt = internalReportDisplay?.prompt || taskPromptText;
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
@@ -433,7 +483,7 @@ class Agent {
         const runAiAct = async ()=>{
             const planningModel = this.resolveModelRuntime('planning');
             const defaultModel = this.resolveModelRuntime('default');
-            const aiActContext = opt?.context !== void 0 ? opt.context : this.aiActContext;
+            const aiActContext = this.resolveContext('aiAct', opt?.context, internalOptions?._internalAdditionalContext);
             const cachePrompt = (0, external_prompt_context_js_namespaceObject.buildPromptWithContext)(taskPrompt, aiActContext);
             const effort = (()=>{
                 const resolvedEffort = opt?.effort ?? (opt?.deepThink === true ? 'deepThink' : 'balance');
@@ -551,7 +601,7 @@ class Agent {
         return this.createInsight().aiAsk(prompt, opt);
     }
     async aiLocate(prompt, opt) {
-        const locateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(prompt, opt);
+        const locateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(prompt, this.withContext('aiLocate', opt));
         (0, utils_namespaceObject.assert)(locateParam, 'cannot get locate param for aiLocate');
         const locatePlan = (0, external_tasks_js_namespaceObject.locatePlanForLocate)(locateParam);
         const plans = [
@@ -574,10 +624,11 @@ class Agent {
     }
     async aiWaitFor(assertion, opt) {
         const modelRuntime = this.resolveModelRuntime('insight');
+        const options = this.withContext('aiWaitFor', opt);
         await this.taskExecutor.waitFor(assertion, {
-            ...opt,
-            timeoutMs: opt?.timeoutMs || 15000,
-            checkIntervalMs: opt?.checkIntervalMs || 3000
+            ...options,
+            timeoutMs: options?.timeoutMs || 15000,
+            checkIntervalMs: options?.checkIntervalMs || 3000
         }, modelRuntime);
     }
     async ai(...args) {
@@ -846,11 +897,23 @@ class Agent {
             groupDescription: ''
         }, opts || {});
         (0, external_report_generator_js_namespaceObject.assertReportGenerationOptions)(this.opts);
-        const resolvedAiActContext = this.opts.aiActContext ?? this.opts.aiActionContext;
+        if (void 0 !== this.opts.contexts && ('object' != typeof this.opts.contexts || null === this.opts.contexts || Array.isArray(this.opts.contexts))) throw new TypeError('opts.contexts must be a plain object');
+        for (const [key, value] of Object.entries(this.opts.contexts ?? {})){
+            if (!(0, agent_context_namespaceObject.isAgentContextKey)(key)) throw new TypeError(`Unknown Agent context key: ${key}`);
+            if (void 0 !== value && 'string' != typeof value) throw new TypeError(`Agent context "${key}" must be a string`);
+        }
+        const deprecatedAiActContextOption = void 0 !== this.opts.aiActContext ? 'aiActContext' : void 0 !== this.opts.aiActionContext ? 'aiActionContext' : void 0;
+        if (deprecatedAiActContextOption) warn(`Agent option "${deprecatedAiActContextOption}" is deprecated; use "contexts.aiAct" instead. When both are provided, "contexts.aiAct" takes precedence.`);
+        const normalizedContexts = {
+            ...this.opts.contexts
+        };
+        const resolvedAiActContext = normalizedContexts.aiAct ?? this.opts.aiActContext ?? this.opts.aiActionContext;
         if (void 0 !== resolvedAiActContext) {
+            normalizedContexts.aiAct = resolvedAiActContext;
             this.opts.aiActContext = resolvedAiActContext;
             this.opts.aiActionContext ??= resolvedAiActContext;
         }
+        this.opts.contexts = normalizedContexts;
         if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
         const hasCustomConfig = opts?.modelConfig || opts?.createOpenAIClient;
         this.modelConfigManager = hasCustomConfig ? new env_namespaceObject.ModelConfigManager(opts?.modelConfig, opts?.createOpenAIClient) : env_namespaceObject.globalModelConfigManager;