@midscene/core 1.17.4 → 1.17.5-beta-20261102094111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +20 -4
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/plan-freshness.mjs +39 -0
- package/dist/es/agent/plan-freshness.mjs.map +1 -0
- package/dist/es/agent/planned-wait.mjs +62 -0
- package/dist/es/agent/planned-wait.mjs.map +1 -0
- package/dist/es/agent/tasks.mjs +103 -57
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
- package/dist/es/report-html-template.mjs +1 -1
- package/dist/es/task-runner.mjs +16 -1
- package/dist/es/task-runner.mjs.map +1 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +1 -1
- package/dist/lib/agent/agent.js +20 -4
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/plan-freshness.js +76 -0
- package/dist/lib/agent/plan-freshness.js.map +1 -0
- package/dist/lib/agent/planned-wait.js +105 -0
- package/dist/lib/agent/planned-wait.js.map +1 -0
- package/dist/lib/agent/tasks.js +102 -56
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/utils.js +1 -1
- package/dist/lib/ai-model/workflows/planning/custom-planning.js +1 -1
- package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
- package/dist/lib/ai-model/workflows/planning/standard-planning.js +1 -1
- package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
- package/dist/lib/report-html-template.js +1 -1
- package/dist/lib/task-runner.js +15 -0
- package/dist/lib/task-runner.js.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/types/agent/agent.d.ts +9 -0
- package/dist/types/agent/agent.d.ts.map +1 -1
- package/dist/types/agent/plan-freshness.d.ts +15 -0
- package/dist/types/agent/plan-freshness.d.ts.map +1 -0
- package/dist/types/agent/planned-wait.d.ts +11 -0
- package/dist/types/agent/planned-wait.d.ts.map +1 -0
- package/dist/types/agent/tasks.d.ts +3 -1
- package/dist/types/agent/tasks.d.ts.map +1 -1
- package/dist/types/task-runner.d.ts +1 -0
- package/dist/types/task-runner.d.ts.map +1 -1
- package/dist/types/types.d.ts +10 -0
- package/dist/types/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __webpack_require__ = {};
|
|
3
|
+
(()=>{
|
|
4
|
+
__webpack_require__.d = (exports1, definition)=>{
|
|
5
|
+
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
6
|
+
enumerable: true,
|
|
7
|
+
get: definition[key]
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
})();
|
|
11
|
+
(()=>{
|
|
12
|
+
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
13
|
+
})();
|
|
14
|
+
(()=>{
|
|
15
|
+
__webpack_require__.r = (exports1)=>{
|
|
16
|
+
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
17
|
+
value: 'Module'
|
|
18
|
+
});
|
|
19
|
+
Object.defineProperty(exports1, '__esModule', {
|
|
20
|
+
value: true
|
|
21
|
+
});
|
|
22
|
+
};
|
|
23
|
+
})();
|
|
24
|
+
var __webpack_exports__ = {};
|
|
25
|
+
__webpack_require__.r(__webpack_exports__);
|
|
26
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
+
PlannedWaitError: ()=>PlannedWaitError,
|
|
28
|
+
definePlannedWait: ()=>definePlannedWait,
|
|
29
|
+
plannedWaitContext: ()=>plannedWaitContext,
|
|
30
|
+
isPlannedWaitError: ()=>isPlannedWaitError
|
|
31
|
+
});
|
|
32
|
+
const index_js_namespaceObject = require("../device/index.js");
|
|
33
|
+
const external_zod_namespaceObject = require("zod");
|
|
34
|
+
function _define_property(obj, key, value) {
|
|
35
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
36
|
+
value: value,
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true
|
|
40
|
+
});
|
|
41
|
+
else obj[key] = value;
|
|
42
|
+
return obj;
|
|
43
|
+
}
|
|
44
|
+
const plannedWaitContext = `Conditional waiting policy for all UI tasks:
|
|
45
|
+
- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as "the interface may take time to respond". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.
|
|
46
|
+
- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.
|
|
47
|
+
- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.
|
|
48
|
+
- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.
|
|
49
|
+
- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.
|
|
50
|
+
- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.
|
|
51
|
+
- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.
|
|
52
|
+
- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.
|
|
53
|
+
- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;
|
|
54
|
+
class PlannedWaitError extends Error {
|
|
55
|
+
constructor(cause){
|
|
56
|
+
super(`Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`, {
|
|
57
|
+
cause
|
|
58
|
+
}), _define_property(this, "code", 'PLANNED_WAIT_FAILED');
|
|
59
|
+
this.name = 'PlannedWaitError';
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
function isPlannedWaitError(error) {
|
|
63
|
+
const seen = new Set();
|
|
64
|
+
while(error && 'object' == typeof error && !seen.has(error)){
|
|
65
|
+
seen.add(error);
|
|
66
|
+
const value = error;
|
|
67
|
+
if ('PLANNED_WAIT_FAILED' === value.code) return true;
|
|
68
|
+
error = value.cause;
|
|
69
|
+
}
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
function definePlannedWait(wait) {
|
|
73
|
+
return (0, index_js_namespaceObject.defineAction)({
|
|
74
|
+
name: 'WaitFor',
|
|
75
|
+
description: 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',
|
|
76
|
+
paramSchema: external_zod_namespaceObject.z.object({
|
|
77
|
+
condition: external_zod_namespaceObject.z.string().trim().min(1).describe('The visible condition needed before continuing, which can become true without another user action.')
|
|
78
|
+
}),
|
|
79
|
+
delayBeforeRunner: 0,
|
|
80
|
+
delayAfterRunner: 0,
|
|
81
|
+
call: async ({ condition }, context)=>{
|
|
82
|
+
try {
|
|
83
|
+
await wait(condition, context?.signal);
|
|
84
|
+
if (context) context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;
|
|
85
|
+
} catch (error) {
|
|
86
|
+
throw new PlannedWaitError(error);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
exports.PlannedWaitError = __webpack_exports__.PlannedWaitError;
|
|
92
|
+
exports.definePlannedWait = __webpack_exports__.definePlannedWait;
|
|
93
|
+
exports.isPlannedWaitError = __webpack_exports__.isPlannedWaitError;
|
|
94
|
+
exports.plannedWaitContext = __webpack_exports__.plannedWaitContext;
|
|
95
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
96
|
+
"PlannedWaitError",
|
|
97
|
+
"definePlannedWait",
|
|
98
|
+
"isPlannedWaitError",
|
|
99
|
+
"plannedWaitContext"
|
|
100
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
101
|
+
Object.defineProperty(exports, '__esModule', {
|
|
102
|
+
value: true
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
//# sourceMappingURL=planned-wait.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent/planned-wait.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/planned-wait.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defineAction } from '@/device';\nimport { z } from 'zod';\n\n/** Planning guidance is only attached when conditional waiting is enabled. */\nexport const plannedWaitContext = `Conditional waiting policy for all UI tasks:\n- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as \"the interface may take time to respond\". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.\n- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.\n- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.\n- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.\n- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.\n- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.\n- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.\n- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.\n- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;\n\nexport class PlannedWaitError extends Error {\n readonly code = 'PLANNED_WAIT_FAILED';\n constructor(cause: unknown) {\n super(\n `Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`,\n { cause },\n );\n this.name = 'PlannedWaitError';\n }\n}\n\nexport function isPlannedWaitError(error: unknown): boolean {\n const seen = new Set<unknown>();\n while (error && typeof error === 'object' && !seen.has(error)) {\n seen.add(error);\n const value = error as { code?: unknown; cause?: unknown };\n if (value.code === 'PLANNED_WAIT_FAILED') return true;\n error = value.cause;\n }\n return false;\n}\n\nexport function definePlannedWait(\n wait: (condition: string, signal?: AbortSignal) => Promise<void>,\n) {\n return defineAction({\n name: 'WaitFor',\n description:\n 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',\n paramSchema: z.object({\n condition: z\n .string()\n .trim()\n .min(1)\n .describe(\n 'The visible condition needed before continuing, which can become true without another user action.',\n ),\n }),\n delayBeforeRunner: 0,\n delayAfterRunner: 0,\n call: async ({ condition }, context) => {\n try {\n await wait(condition, context?.signal);\n if (context) {\n context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;\n }\n } catch (error) {\n throw new PlannedWaitError(error);\n }\n },\n });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","plannedWaitContext","PlannedWaitError","Error","cause","String","isPlannedWaitError","error","seen","Set","value","definePlannedWait","wait","defineAction","z","condition","context"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;ACFO,MAAMI,qBAAqB,CAAC;;;;;;;;;6JAS0H,CAAC;AAEvJ,MAAMC,yBAAyBC;IAEpC,YAAYC,KAAc,CAAE;QAC1B,KAAK,CACH,CAAC,wBAAwB,EAAEA,iBAAiBD,QAAQC,MAAM,OAAO,GAAGC,OAAOD,QAAQ,EACnF;YAAEA;QAAM,IAJZ,uBAAS,QAAO;QAMd,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEO,SAASE,mBAAmBC,KAAc;IAC/C,MAAMC,OAAO,IAAIC;IACjB,MAAOF,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,KAAK,GAAG,CAACD,OAAQ;QAC7DC,KAAK,GAAG,CAACD;QACT,MAAMG,QAAQH;QACd,IAAIG,AAAe,0BAAfA,MAAM,IAAI,EAA4B,OAAO;QACjDH,QAAQG,MAAM,KAAK;IACrB;IACA,OAAO;AACT;AAEO,SAASC,kBACdC,IAAgE;IAEhE,OAAOC,AAAAA,IAAAA,yBAAAA,YAAAA,AAAAA,EAAa;QAClB,MAAM;QACN,aACE;QACF,aAAaC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;YACpB,WAAWA,6BAAAA,CAAAA,CAAAA,MACF,GACN,IAAI,GACJ,GAAG,CAAC,GACJ,QAAQ,CACP;QAEN;QACA,mBAAmB;QACnB,kBAAkB;QAClB,MAAM,OAAO,EAAEC,SAAS,EAAE,EAAEC;YAC1B,IAAI;gBACF,MAAMJ,KAAKG,WAAWC,SAAS;gBAC/B,IAAIA,SACFA,QAAQ,IAAI,CAAC,gBAAgB,GAAG,CAAC,mBAAmB,EAAED,UAAU,2HAA2H,CAAC;YAEhM,EAAE,OAAOR,OAAO;gBACd,MAAM,IAAIL,iBAAiBK;YAC7B;QACF;IACF;AACF"}
|
package/dist/lib/agent/tasks.js
CHANGED
|
@@ -44,6 +44,8 @@ const system_one_index_js_namespaceObject = require("../ai-model/service-caller/
|
|
|
44
44
|
const external_action_readiness_js_namespaceObject = require("./action-readiness.js");
|
|
45
45
|
const external_execution_session_js_namespaceObject = require("./execution-session.js");
|
|
46
46
|
const external_file_chooser_js_namespaceObject = require("./file-chooser.js");
|
|
47
|
+
const external_plan_freshness_js_namespaceObject = require("./plan-freshness.js");
|
|
48
|
+
const external_planned_wait_js_namespaceObject = require("./planned-wait.js");
|
|
47
49
|
const external_progress_index_js_namespaceObject = require("./progress/index.js");
|
|
48
50
|
const external_prompt_context_js_namespaceObject = require("./prompt-context.js");
|
|
49
51
|
const external_task_builder_js_namespaceObject = require("./task-builder.js");
|
|
@@ -217,9 +219,9 @@ class TaskExecutor {
|
|
|
217
219
|
runner
|
|
218
220
|
};
|
|
219
221
|
}
|
|
220
|
-
async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
|
|
222
|
+
async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
|
|
221
223
|
abortSignal?.throwIfAborted();
|
|
222
|
-
return (0, external_file_chooser_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
|
|
224
|
+
return (0, external_file_chooser_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions, checkScreenChanges));
|
|
223
225
|
}
|
|
224
226
|
invalidateFailedCacheHitLocates(runner, fromIndex) {
|
|
225
227
|
if (!this.taskCache) return;
|
|
@@ -231,7 +233,7 @@ class TaskExecutor {
|
|
|
231
233
|
}
|
|
232
234
|
}
|
|
233
235
|
}
|
|
234
|
-
async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
|
|
236
|
+
async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
|
|
235
237
|
abortSignal ??= this.getAbortSignal?.();
|
|
236
238
|
const conversationHistory = new index_js_namespaceObject.ConversationHistory();
|
|
237
239
|
const promptDisplay = reportOptions?.prompt || (0, external_common_js_namespaceObject.userPromptToString)(userPrompt);
|
|
@@ -264,6 +266,8 @@ class TaskExecutor {
|
|
|
264
266
|
noIndividualLocateModel
|
|
265
267
|
});
|
|
266
268
|
let replanCount = 0;
|
|
269
|
+
let consecutiveScreenChangeReplans = 0;
|
|
270
|
+
const screenChangeReplanLimit = 2;
|
|
267
271
|
const yamlFlow = [];
|
|
268
272
|
const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
|
|
269
273
|
if (void 0 === replanningCycleLimit) throw new Error('replanningCycleLimit is required for TaskExecutor.action');
|
|
@@ -394,6 +398,14 @@ class TaskExecutor {
|
|
|
394
398
|
} finally{
|
|
395
399
|
(0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callAiEnd');
|
|
396
400
|
}
|
|
401
|
+
if (this.planningWaitFor && !planResult.delegation) {
|
|
402
|
+
const waitIndex = planResult.actions?.findIndex((action)=>'WaitFor' === action.type) ?? -1;
|
|
403
|
+
if (waitIndex >= 0) {
|
|
404
|
+
planResult.actions = planResult.actions.slice(0, waitIndex + 1);
|
|
405
|
+
planResult.yamlFlow = (0, external_common_js_namespaceObject.buildYamlFlowFromPlans)(planResult.actions, actionSpace);
|
|
406
|
+
planResult.shouldContinuePlanning = true;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
397
409
|
debug('planResult', JSON.stringify(planResult, null, 2));
|
|
398
410
|
const { actions, thought, log, memory, error, usage, rawResponse, rawAssistantOutput, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
|
|
399
411
|
outputString = finalizeMessage;
|
|
@@ -456,7 +468,18 @@ class TaskExecutor {
|
|
|
456
468
|
});
|
|
457
469
|
const planResult = result?.output;
|
|
458
470
|
latestPlanResult = planResult;
|
|
459
|
-
|
|
471
|
+
let freshness;
|
|
472
|
+
if (checkScreenChanges && !planResult?.delegation) try {
|
|
473
|
+
freshness = await (0, external_plan_freshness_js_namespaceObject.evaluatePlanFreshness)(runner.tasks[runner.tasks.length - 1], planResult?.actions || [], runner.id, planIndex);
|
|
474
|
+
} catch (error) {
|
|
475
|
+
return appendFailedPlan(`Failed to check screen freshness: ${error instanceof Error ? error.message : String(error)}`, planIndex);
|
|
476
|
+
}
|
|
477
|
+
if (freshness?.changed) {
|
|
478
|
+
if (consecutiveScreenChangeReplans >= screenChangeReplanLimit) return appendFailedPlan(`Task stopped: the screen kept changing during planning after ${screenChangeReplanLimit} consecutive screen-change replans. No action from the rejected plans was executed. Wait for the page to settle before trying again.`, planIndex);
|
|
479
|
+
++consecutiveScreenChangeReplans;
|
|
480
|
+
this.setPendingFeedbackMessage(conversationHistory, await this.getTimeString(), 'The planned action was NOT executed: the screen changed while planning. Inspect the current screenshot and plan again; do not assume the previous action happened.');
|
|
481
|
+
}
|
|
482
|
+
if (!planResult?.delegation && !freshness?.changed) {
|
|
460
483
|
const plans = planResult?.actions || [];
|
|
461
484
|
yamlFlow.push(...planResult?.yamlFlow || []);
|
|
462
485
|
let executables;
|
|
@@ -476,9 +499,10 @@ class TaskExecutor {
|
|
|
476
499
|
activeActionReporter = (0, external_progress_index_js_namespaceObject.createAiActActionReporter)(planIndex, replanningCycleLimit, (phase, data)=>this.emitAiActProgress(phase, data));
|
|
477
500
|
try {
|
|
478
501
|
await session.appendAndRun(executables.tasks);
|
|
502
|
+
if (executables.tasks.some((task)=>'Action Space' === task.type)) consecutiveScreenChangeReplans = 0;
|
|
479
503
|
this.setPendingFeedbackMessage(conversationHistory, initialTimeString, this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)));
|
|
480
504
|
} catch (error) {
|
|
481
|
-
if ((0, external_action_readiness_js_namespaceObject.isActionReadinessError)(error)) throw error;
|
|
505
|
+
if ((0, external_action_readiness_js_namespaceObject.isActionReadinessError)(error) || (0, external_planned_wait_js_namespaceObject.isPlannedWaitError)(error)) throw error;
|
|
482
506
|
errorCountInOnePlanningLoop++;
|
|
483
507
|
const timeString = await this.getTimeString();
|
|
484
508
|
this.setPendingFeedbackMessage(conversationHistory, timeString, `Error executing running tasks: ${error?.message || String(error)}`);
|
|
@@ -632,64 +656,84 @@ class TaskExecutor {
|
|
|
632
656
|
};
|
|
633
657
|
}
|
|
634
658
|
async waitFor(assertion, opt, modelRuntime) {
|
|
635
|
-
const
|
|
636
|
-
const
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
abortSignal
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
const
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
const
|
|
648
|
-
const
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
const overallStartTime = Date.now();
|
|
658
|
-
let lastCheckStart = overallStartTime;
|
|
659
|
-
let errorThought = '';
|
|
660
|
-
while(lastCheckStart - overallStartTime <= timeoutMs){
|
|
661
|
-
const currentCheckStart = Date.now();
|
|
662
|
-
lastCheckStart = currentCheckStart;
|
|
663
|
-
abortSignal?.throwIfAborted();
|
|
664
|
-
const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
|
|
659
|
+
const timeoutMs = opt.timeoutMs;
|
|
660
|
+
const checkIntervalMs = opt.checkIntervalMs;
|
|
661
|
+
(0, utils_namespaceObject.assert)('number' == typeof timeoutMs && Number.isFinite(timeoutMs) && timeoutMs > 0 && timeoutMs <= 2147483647, 'waitFor timeoutMs must be a finite positive timer duration');
|
|
662
|
+
(0, utils_namespaceObject.assert)('number' == typeof checkIntervalMs && Number.isFinite(checkIntervalMs) && checkIntervalMs > 0 && checkIntervalMs <= timeoutMs, 'waitFor checkIntervalMs must be positive and no greater than timeoutMs');
|
|
663
|
+
const signals = [
|
|
664
|
+
opt.abortSignal,
|
|
665
|
+
this.getAbortSignal?.()
|
|
666
|
+
].filter((signal)=>void 0 !== signal);
|
|
667
|
+
const callerSignal = signals.length ? AbortSignal.any(signals) : void 0;
|
|
668
|
+
callerSignal?.throwIfAborted();
|
|
669
|
+
const deadline = new AbortController();
|
|
670
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
671
|
+
const timer = setTimeout(()=>deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`)), timeoutMs);
|
|
672
|
+
const abortSignal = callerSignal ? AbortSignal.any([
|
|
673
|
+
callerSignal,
|
|
674
|
+
deadline.signal
|
|
675
|
+
]) : deadline.signal;
|
|
676
|
+
try {
|
|
677
|
+
const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject.parsePrompt)(assertion);
|
|
678
|
+
const description = `waitFor: ${textPrompt}`;
|
|
679
|
+
const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)('WaitFor', description), {
|
|
680
|
+
referenceImages: multimodalPrompt?.images,
|
|
665
681
|
abortSignal
|
|
666
682
|
});
|
|
667
|
-
const
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
runner
|
|
683
|
+
const runner = session.getRunner();
|
|
684
|
+
const executionModelRuntime = {
|
|
685
|
+
...modelRuntime,
|
|
686
|
+
executionId: runner.id
|
|
671
687
|
};
|
|
672
|
-
|
|
673
|
-
const
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
}
|
|
686
|
-
], modelRuntime, modelRuntime, {
|
|
688
|
+
const { timeoutMs: _timeoutMs, checkIntervalMs: _checkIntervalMs, abortSignal: _abortSignal, domIncluded, screenshotIncluded, ...restOpt } = opt;
|
|
689
|
+
const serviceExtractOpt = {
|
|
690
|
+
domIncluded,
|
|
691
|
+
screenshotIncluded,
|
|
692
|
+
...restOpt
|
|
693
|
+
};
|
|
694
|
+
(0, utils_namespaceObject.assert)(assertion, 'No assertion for waitFor');
|
|
695
|
+
const overallStartTime = Date.now();
|
|
696
|
+
let errorThought = '';
|
|
697
|
+
while(Date.now() - overallStartTime < timeoutMs){
|
|
698
|
+
const currentCheckStart = Date.now();
|
|
699
|
+
abortSignal?.throwIfAborted();
|
|
700
|
+
const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
|
|
687
701
|
abortSignal
|
|
688
702
|
});
|
|
689
|
-
|
|
703
|
+
const result = await session.appendAndRun(queryTask);
|
|
704
|
+
abortSignal.throwIfAborted();
|
|
705
|
+
if (Date.now() >= deadlineAt) {
|
|
706
|
+
deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`));
|
|
707
|
+
abortSignal.throwIfAborted();
|
|
708
|
+
}
|
|
709
|
+
if (result?.output) return {
|
|
710
|
+
output: void 0,
|
|
711
|
+
runner
|
|
712
|
+
};
|
|
713
|
+
errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
|
|
714
|
+
const now = Date.now();
|
|
715
|
+
if (now - currentCheckStart < checkIntervalMs) {
|
|
716
|
+
const elapsed = now - currentCheckStart;
|
|
717
|
+
const timeRemaining = Math.max(0, Math.min(checkIntervalMs - elapsed, timeoutMs - (Date.now() - overallStartTime)));
|
|
718
|
+
const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
|
|
719
|
+
const { tasks: sleepTasks } = await this.convertPlanToExecutable([
|
|
720
|
+
{
|
|
721
|
+
type: 'Sleep',
|
|
722
|
+
param: {
|
|
723
|
+
timeMs: timeRemaining
|
|
724
|
+
},
|
|
725
|
+
thought
|
|
726
|
+
}
|
|
727
|
+
], modelRuntime, modelRuntime, {
|
|
728
|
+
abortSignal
|
|
729
|
+
});
|
|
730
|
+
if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
|
|
731
|
+
}
|
|
690
732
|
}
|
|
733
|
+
return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
|
|
734
|
+
} finally{
|
|
735
|
+
clearTimeout(timer);
|
|
691
736
|
}
|
|
692
|
-
return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
|
|
693
737
|
}
|
|
694
738
|
constructor(interfaceInstance, service, opts){
|
|
695
739
|
_define_property(this, "interface", void 0);
|
|
@@ -697,6 +741,7 @@ class TaskExecutor {
|
|
|
697
741
|
_define_property(this, "taskCache", void 0);
|
|
698
742
|
_define_property(this, "getDefaultModelRuntime", void 0);
|
|
699
743
|
_define_property(this, "providedActionSpace", void 0);
|
|
744
|
+
_define_property(this, "planningWaitFor", void 0);
|
|
700
745
|
_define_property(this, "taskBuilder", void 0);
|
|
701
746
|
_define_property(this, "onTaskStartCallback", void 0);
|
|
702
747
|
_define_property(this, "hooks", void 0);
|
|
@@ -710,6 +755,7 @@ class TaskExecutor {
|
|
|
710
755
|
this.taskCache = opts.taskCache;
|
|
711
756
|
this.onTaskStartCallback = opts?.onTaskStart;
|
|
712
757
|
this.replanningCycleLimit = opts.replanningCycleLimit;
|
|
758
|
+
this.planningWaitFor = opts.planningWaitFor ?? false;
|
|
713
759
|
this.waitAfterAction = opts.waitAfterAction;
|
|
714
760
|
this.useDeviceTime = opts.useDeviceTime;
|
|
715
761
|
this.hooks = opts.hooks;
|