@midscene/core 1.12.3 → 1.12.4-beta-20260902093735.0

package/dist/es/agent/agent.mjs

@@ -10,6 +10,7 @@ import { getVersion, processCacheConfig, reportHTMLContent } from "../utils.mjs"
 import { ScriptPlayer, buildDetailedLocateParam, buildDetailedLocateParamAndRestParams, parseYamlScript } from "../yaml/index.mjs";
 import { readFile } from "node:fs/promises";
 import { basename, resolve as external_node_path_resolve } from "node:path";
+import { isAgentContextKey } from "@midscene/shared/agent-tools/agent-context";
 import { serializeError } from "@midscene/shared/agent-tools/error-formatter";
 import { observationArtifactAdapterSymbol } from "@midscene/shared/agent-tools/observation-artifact";
 import { MIDSCENE_REPLANNING_CYCLE_LIMIT, ModelConfigManager, globalConfigManager, globalModelConfigManager } from "@midscene/shared/env";
@@ -21,7 +22,7 @@ import { FileChooserAccepter } from "./file-chooser.mjs";
 import { Insight } from "./insight.mjs";
 import { MetricsCollector } from "./metrics.mjs";
 import { AgentProgressBus } from "./progress/index.mjs";
-import { buildPromptWithContext } from "./prompt-context.mjs";
+import { INTERNAL_AI_CONTEXT_METADATA_KEY, buildPromptWithContext, renderAIContext } from "./prompt-context.mjs";
 import { normalizeRecordToReportScreenshot } from "./record-to-report.mjs";
 import { runGherkinScenario } from "./run-gherkin-scenario.mjs";
 import { markdownToAiActPrompt } from "./run-markdown.mjs";
@@ -52,8 +53,44 @@ class Agent {
         this.dumpUpdateListeners = [];
         if (callback) this.dumpUpdateListeners.push(callback);
     }
-    get aiActContext() {
-        return this.opts.aiActContext ?? this.opts.aiActionContext;
+    get contexts() {
+        if (!this.opts.contexts) this.opts.contexts = {};
+        return this.opts.contexts;
+    }
+    resolveUserContext(apiName, callContext) {
+        if (void 0 !== callContext) return {
+            value: callContext,
+            metadata: {
+                source: 'call'
+            }
+        };
+        const apiContext = this.opts.contexts?.[apiName];
+        if (void 0 !== apiContext) return {
+            value: apiContext,
+            metadata: {
+                source: 'api',
+                apiName
+            }
+        };
+        const defaultContext = this.opts.contexts?.default;
+        if (void 0 !== defaultContext) return {
+            value: defaultContext,
+            metadata: {
+                source: 'default'
+            }
+        };
+    }
+    resolveContext(apiName, callContext, additionalContext) {
+        return renderAIContext(this.resolveUserContext(apiName, callContext), additionalContext);
+    }
+    withContext(apiName, options) {
+        const resolvedContext = this.resolveUserContext(apiName, options?.context);
+        if (void 0 === resolvedContext) return options;
+        return {
+            ...options ?? {},
+            context: resolvedContext.value,
+            [INTERNAL_AI_CONTEXT_METADATA_KEY]: resolvedContext.metadata
+        };
     }
     get page() {
         return this.interface;
@@ -79,7 +116,7 @@ class Agent {
         };
     }
     createInsight(getUIContext) {
-        return new Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), getUIContext);
+        return new Insight(this.taskExecutor, ()=>this.resolveModelRuntime('insight'), getUIContext, (apiName, callContext)=>this.resolveUserContext(apiName, callContext));
     }
     async getActionSpace() {
         return this.fullActionSpace;
@@ -140,12 +177,24 @@ class Agent {
         return observer;
     }
     async setAIActionContext(prompt) {
-        await this.setAIActContext(prompt);
+        warn('setAIActionContext() is deprecated; use setContext("aiAct", context) instead.');
+        this.setContext('aiAct', prompt);
     }
     async setAIActContext(prompt) {
-        if (this.aiActContext) console.warn('aiActContext is already set, and it is called again, will override the previous setting');
-        this.opts.aiActContext = prompt;
-        this.opts.aiActionContext = prompt;
+        warn('setAIActContext() is deprecated; use setContext("aiAct", context) instead.');
+        this.setContext('aiAct', prompt);
+    }
+    setContext(target, context) {
+        if (!isAgentContextKey(target)) throw new TypeError(`Unknown Agent context key: ${String(target)}`);
+        if (void 0 !== context && 'string' != typeof context) throw new TypeError('Agent context must be a string or undefined');
+        this.contexts[target] = context;
+        if ('aiAct' === target) if (void 0 === context) {
+            this.opts.aiActContext = void 0;
+            this.opts.aiActionContext = void 0;
+        } else {
+            this.opts.aiActContext = context;
+            this.opts.aiActionContext = context;
+        }
     }
     resetDump() {
         this.dump = new ReportActionDump({
@@ -249,7 +298,7 @@ class Agent {
     }
     async aiTap(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for tap');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiTap', opt));
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
         await withFileChooser(this.interface, fileChooserAccept, async ()=>{
             await this.callActionInActionSpace('Tap', {
@@ -259,21 +308,21 @@ class Agent {
     }
     async aiRightClick(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for right click');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiRightClick', opt));
         await this.callActionInActionSpace('RightClick', {
             locate: detailedLocateParam
         });
     }
     async aiDoubleClick(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for double click');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiDoubleClick', opt));
         await this.callActionInActionSpace('DoubleClick', {
             locate: detailedLocateParam
         });
     }
     async aiHover(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for hover');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiHover', opt));
         await this.callActionInActionSpace('Hover', {
             locate: detailedLocateParam
         });
@@ -297,7 +346,7 @@ class Agent {
         }
         assert('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
         assert(locatePrompt, 'missing locate prompt for input');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, this.withContext('aiInput', opt));
         const stringValue = 'number' == typeof value ? String(value) : value;
         const mode = opt?.mode === 'append' ? 'typeOnly' : opt?.mode;
         await this.callActionInActionSpace('Input', {
@@ -323,7 +372,7 @@ class Agent {
             };
         }
         assert(opt?.keyName, 'missing keyName for keyboard press');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiKeyboardPress', opt));
         await this.callActionInActionSpace('KeyboardPress', {
             ...restParams,
             locate: locateParam
@@ -355,14 +404,14 @@ class Agent {
                 scrollType: normalizedScrollType
             };
         }
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiScroll', opt));
         await this.callActionInActionSpace('Scroll', {
             ...restParams,
             locate: locateParam
         });
     }
     async aiPinch(locatePrompt, opt) {
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt || '', this.withContext('aiPinch', opt));
         await this.callActionInActionSpace('Pinch', {
             ...restParams,
             locate: locateParam
@@ -370,7 +419,7 @@ class Agent {
     }
     async aiLongPress(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for long press');
-        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, opt);
+        const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePrompt, this.withContext('aiLongPress', opt));
         await this.callActionInActionSpace('LongPress', {
             ...restParams,
             locate: locateParam
@@ -378,13 +427,14 @@ class Agent {
     }
     async aiClearInput(locatePrompt, opt) {
         assert(locatePrompt, 'missing locate prompt for clear input');
-        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, opt);
+        const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiClearInput', opt));
         await this.callActionInActionSpace('ClearInput', {
             locate: detailedLocateParam
         });
     }
     async aiAct(taskPrompt, opt) {
-        const internalReportDisplay = opt?._internalReportDisplay;
+        const internalOptions = opt;
+        const internalReportDisplay = internalOptions?._internalReportDisplay;
         const taskPromptText = 'string' == typeof taskPrompt ? taskPrompt : taskPrompt.prompt;
         const reportPrompt = internalReportDisplay?.prompt || taskPromptText;
         const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
@@ -393,7 +443,7 @@ class Agent {
         const runAiAct = async ()=>{
             const planningModel = this.resolveModelRuntime('planning');
             const defaultModel = this.resolveModelRuntime('default');
-            const aiActContext = opt?.context !== void 0 ? opt.context : this.aiActContext;
+            const aiActContext = this.resolveContext('aiAct', opt?.context, internalOptions?._internalAdditionalContext);
             const cachePrompt = buildPromptWithContext(taskPrompt, aiActContext);
             const effort = (()=>{
                 const resolvedEffort = opt?.effort ?? (opt?.deepThink === true ? 'deepThink' : 'balance');
@@ -511,7 +561,7 @@ class Agent {
         return this.createInsight().aiAsk(prompt, opt);
     }
     async aiLocate(prompt, opt) {
-        const locateParam = buildDetailedLocateParam(prompt, opt);
+        const locateParam = buildDetailedLocateParam(prompt, this.withContext('aiLocate', opt));
         assert(locateParam, 'cannot get locate param for aiLocate');
         const locatePlan = locatePlanForLocate(locateParam);
         const plans = [
@@ -534,10 +584,11 @@ class Agent {
     }
     async aiWaitFor(assertion, opt) {
         const modelRuntime = this.resolveModelRuntime('insight');
+        const options = this.withContext('aiWaitFor', opt);
         await this.taskExecutor.waitFor(assertion, {
-            ...opt,
-            timeoutMs: opt?.timeoutMs || 15000,
-            checkIntervalMs: opt?.checkIntervalMs || 3000
+            ...options,
+            timeoutMs: options?.timeoutMs || 15000,
+            checkIntervalMs: options?.checkIntervalMs || 3000
         }, modelRuntime);
     }
     async ai(...args) {
@@ -806,11 +857,23 @@ class Agent {
             groupDescription: ''
         }, opts || {});
         assertReportGenerationOptions(this.opts);
-        const resolvedAiActContext = this.opts.aiActContext ?? this.opts.aiActionContext;
+        if (void 0 !== this.opts.contexts && ('object' != typeof this.opts.contexts || null === this.opts.contexts || Array.isArray(this.opts.contexts))) throw new TypeError('opts.contexts must be a plain object');
+        for (const [key, value] of Object.entries(this.opts.contexts ?? {})){
+            if (!isAgentContextKey(key)) throw new TypeError(`Unknown Agent context key: ${key}`);
+            if (void 0 !== value && 'string' != typeof value) throw new TypeError(`Agent context "${key}" must be a string`);
+        }
+        const deprecatedAiActContextOption = void 0 !== this.opts.aiActContext ? 'aiActContext' : void 0 !== this.opts.aiActionContext ? 'aiActionContext' : void 0;
+        if (deprecatedAiActContextOption) warn(`Agent option "${deprecatedAiActContextOption}" is deprecated; use "contexts.aiAct" instead. When both are provided, "contexts.aiAct" takes precedence.`);
+        const normalizedContexts = {
+            ...this.opts.contexts
+        };
+        const resolvedAiActContext = normalizedContexts.aiAct ?? this.opts.aiActContext ?? this.opts.aiActionContext;
         if (void 0 !== resolvedAiActContext) {
+            normalizedContexts.aiAct = resolvedAiActContext;
             this.opts.aiActContext = resolvedAiActContext;
             this.opts.aiActionContext ??= resolvedAiActContext;
         }
+        this.opts.contexts = normalizedContexts;
         if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
         const hasCustomConfig = opts?.modelConfig || opts?.createOpenAIClient;
         this.modelConfigManager = hasCustomConfig ? new ModelConfigManager(opts?.modelConfig, opts?.createOpenAIClient) : globalModelConfigManager;