@midscene/core 1.17.4 → 1.17.5-beta-20261102094111.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/es/agent/agent.mjs +20 -4
  2. package/dist/es/agent/agent.mjs.map +1 -1
  3. package/dist/es/agent/plan-freshness.mjs +39 -0
  4. package/dist/es/agent/plan-freshness.mjs.map +1 -0
  5. package/dist/es/agent/planned-wait.mjs +62 -0
  6. package/dist/es/agent/planned-wait.mjs.map +1 -0
  7. package/dist/es/agent/tasks.mjs +103 -57
  8. package/dist/es/agent/tasks.mjs.map +1 -1
  9. package/dist/es/agent/utils.mjs +1 -1
  10. package/dist/es/ai-model/workflows/planning/custom-planning.mjs +1 -1
  11. package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
  12. package/dist/es/ai-model/workflows/planning/standard-planning.mjs +1 -1
  13. package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
  14. package/dist/es/report-html-template.mjs +1 -1
  15. package/dist/es/task-runner.mjs +16 -1
  16. package/dist/es/task-runner.mjs.map +1 -1
  17. package/dist/es/types.mjs.map +1 -1
  18. package/dist/es/utils.mjs +1 -1
  19. package/dist/lib/agent/agent.js +20 -4
  20. package/dist/lib/agent/agent.js.map +1 -1
  21. package/dist/lib/agent/plan-freshness.js +76 -0
  22. package/dist/lib/agent/plan-freshness.js.map +1 -0
  23. package/dist/lib/agent/planned-wait.js +105 -0
  24. package/dist/lib/agent/planned-wait.js.map +1 -0
  25. package/dist/lib/agent/tasks.js +102 -56
  26. package/dist/lib/agent/tasks.js.map +1 -1
  27. package/dist/lib/agent/utils.js +1 -1
  28. package/dist/lib/ai-model/workflows/planning/custom-planning.js +1 -1
  29. package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
  30. package/dist/lib/ai-model/workflows/planning/standard-planning.js +1 -1
  31. package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
  32. package/dist/lib/report-html-template.js +1 -1
  33. package/dist/lib/task-runner.js +15 -0
  34. package/dist/lib/task-runner.js.map +1 -1
  35. package/dist/lib/types.js.map +1 -1
  36. package/dist/lib/utils.js +1 -1
  37. package/dist/types/agent/agent.d.ts +9 -0
  38. package/dist/types/agent/agent.d.ts.map +1 -1
  39. package/dist/types/agent/plan-freshness.d.ts +15 -0
  40. package/dist/types/agent/plan-freshness.d.ts.map +1 -0
  41. package/dist/types/agent/planned-wait.d.ts +11 -0
  42. package/dist/types/agent/planned-wait.d.ts.map +1 -0
  43. package/dist/types/agent/tasks.d.ts +3 -1
  44. package/dist/types/agent/tasks.d.ts.map +1 -1
  45. package/dist/types/task-runner.d.ts +1 -0
  46. package/dist/types/task-runner.d.ts.map +1 -1
  47. package/dist/types/types.d.ts +10 -0
  48. package/dist/types/types.d.ts.map +1 -1
  49. package/package.json +2 -2
@@ -0,0 +1,105 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ PlannedWaitError: ()=>PlannedWaitError,
28
+ definePlannedWait: ()=>definePlannedWait,
29
+ plannedWaitContext: ()=>plannedWaitContext,
30
+ isPlannedWaitError: ()=>isPlannedWaitError
31
+ });
32
+ const index_js_namespaceObject = require("../device/index.js");
33
+ const external_zod_namespaceObject = require("zod");
34
+ function _define_property(obj, key, value) {
35
+ if (key in obj) Object.defineProperty(obj, key, {
36
+ value: value,
37
+ enumerable: true,
38
+ configurable: true,
39
+ writable: true
40
+ });
41
+ else obj[key] = value;
42
+ return obj;
43
+ }
44
+ const plannedWaitContext = `Conditional waiting policy for all UI tasks:
45
+ - PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as "the interface may take time to respond". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.
46
+ - If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.
47
+ - Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.
48
+ - Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.
49
+ - Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.
50
+ - Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.
51
+ - An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.
52
+ - WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.
53
+ - Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;
54
+ class PlannedWaitError extends Error {
55
+ constructor(cause){
56
+ super(`Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`, {
57
+ cause
58
+ }), _define_property(this, "code", 'PLANNED_WAIT_FAILED');
59
+ this.name = 'PlannedWaitError';
60
+ }
61
+ }
62
+ function isPlannedWaitError(error) {
63
+ const seen = new Set();
64
+ while(error && 'object' == typeof error && !seen.has(error)){
65
+ seen.add(error);
66
+ const value = error;
67
+ if ('PLANNED_WAIT_FAILED' === value.code) return true;
68
+ error = value.cause;
69
+ }
70
+ return false;
71
+ }
72
+ function definePlannedWait(wait) {
73
+ return (0, index_js_namespaceObject.defineAction)({
74
+ name: 'WaitFor',
75
+ description: 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',
76
+ paramSchema: external_zod_namespaceObject.z.object({
77
+ condition: external_zod_namespaceObject.z.string().trim().min(1).describe('The visible condition needed before continuing, which can become true without another user action.')
78
+ }),
79
+ delayBeforeRunner: 0,
80
+ delayAfterRunner: 0,
81
+ call: async ({ condition }, context)=>{
82
+ try {
83
+ await wait(condition, context?.signal);
84
+ if (context) context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;
85
+ } catch (error) {
86
+ throw new PlannedWaitError(error);
87
+ }
88
+ }
89
+ });
90
+ }
91
+ exports.PlannedWaitError = __webpack_exports__.PlannedWaitError;
92
+ exports.definePlannedWait = __webpack_exports__.definePlannedWait;
93
+ exports.isPlannedWaitError = __webpack_exports__.isPlannedWaitError;
94
+ exports.plannedWaitContext = __webpack_exports__.plannedWaitContext;
95
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
96
+ "PlannedWaitError",
97
+ "definePlannedWait",
98
+ "isPlannedWaitError",
99
+ "plannedWaitContext"
100
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
101
+ Object.defineProperty(exports, '__esModule', {
102
+ value: true
103
+ });
104
+
105
+ //# sourceMappingURL=planned-wait.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agent/planned-wait.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/planned-wait.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defineAction } from '@/device';\nimport { z } from 'zod';\n\n/** Planning guidance is only attached when conditional waiting is enabled. */\nexport const plannedWaitContext = `Conditional waiting policy for all UI tasks:\n- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as \"the interface may take time to respond\". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.\n- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.\n- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.\n- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.\n- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.\n- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.\n- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.\n- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.\n- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;\n\nexport class PlannedWaitError extends Error {\n readonly code = 'PLANNED_WAIT_FAILED';\n constructor(cause: unknown) {\n super(\n `Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`,\n { cause },\n );\n this.name = 'PlannedWaitError';\n }\n}\n\nexport function isPlannedWaitError(error: unknown): boolean {\n const seen = new Set<unknown>();\n while (error && typeof error === 'object' && !seen.has(error)) {\n seen.add(error);\n const value = error as { code?: unknown; cause?: unknown };\n if (value.code === 'PLANNED_WAIT_FAILED') return true;\n error = value.cause;\n }\n return false;\n}\n\nexport function definePlannedWait(\n wait: (condition: string, signal?: AbortSignal) => Promise<void>,\n) {\n return defineAction({\n name: 'WaitFor',\n description:\n 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',\n paramSchema: z.object({\n condition: z\n .string()\n .trim()\n .min(1)\n .describe(\n 'The visible condition needed before continuing, which can become true without another user action.',\n ),\n }),\n delayBeforeRunner: 0,\n delayAfterRunner: 0,\n call: async ({ condition }, context) => {\n try {\n await wait(condition, context?.signal);\n if (context) {\n context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;\n }\n } catch (error) {\n throw new PlannedWaitError(error);\n }\n },\n });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","plannedWaitContext","PlannedWaitError","Error","cause","String","isPlannedWaitError","error","seen","Set","value","definePlannedWait","wait","defineAction","z","condition","context"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;ACFO,MAAMI,qBAAqB,CAAC;;;;;;;;;6JAS0H,CAAC;AAEvJ,MAAMC,yBAAyBC;IAEpC,YAAYC,KAAc,CAAE;QAC1B,KAAK,CACH,CAAC,wBAAwB,EAAEA,iBAAiBD,QAAQC,MAAM,OAAO,GAAGC,OAAOD,QAAQ,EACnF;YAAEA;QAAM,IAJZ,uBAAS,QAAO;QAMd,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEO,SAASE,mBAAmBC,KAAc;IAC/C,MAAMC,OAAO,IAAIC;IACjB,MAAOF,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,KAAK,GAAG,CAACD,OAAQ;QAC7DC,KAAK,GAAG,CAACD;QACT,MAAMG,QAAQH;QACd,IAAIG,AAAe,0BAAfA,MAAM,IAAI,EAA4B,OAAO;QACjDH,QAAQG,MAAM,KAAK;IACrB;IACA,OAAO;AACT;AAEO,SAASC,kBACdC,IAAgE;IAEhE,OAAOC,AAAAA,IAAAA,yBAAAA,YAAAA,AAAAA,EAAa;QAClB,MAAM;QACN,aACE;QACF,aAAaC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;YACpB,WAAWA,6BAAAA,CAAAA,CAAAA,MACF,GACN,IAAI,GACJ,GAAG,CAAC,GACJ,QAAQ,CACP;QAEN;QACA,mBAAmB;QACnB,kBAAkB;QAClB,MAAM,OAAO,EAAEC,SAAS,EAAE,EAAEC;YAC1B,IAAI;gBACF,MAAMJ,KAAKG,WAAWC,SAAS;gBAC/B,IAAIA,SACFA,QAAQ,IAAI,CAAC,gBAAgB,GAAG,CAAC,mBAAmB,EAAED,UAAU,2HAA2H,CAAC;YAEhM,EAAE,OAAOR,OAAO;gBACd,MAAM,IAAIL,iBAAiBK;YAC7B;QACF;IACF;AACF"}
@@ -44,6 +44,8 @@ const system_one_index_js_namespaceObject = require("../ai-model/service-caller/
44
44
  const external_action_readiness_js_namespaceObject = require("./action-readiness.js");
45
45
  const external_execution_session_js_namespaceObject = require("./execution-session.js");
46
46
  const external_file_chooser_js_namespaceObject = require("./file-chooser.js");
47
+ const external_plan_freshness_js_namespaceObject = require("./plan-freshness.js");
48
+ const external_planned_wait_js_namespaceObject = require("./planned-wait.js");
47
49
  const external_progress_index_js_namespaceObject = require("./progress/index.js");
48
50
  const external_prompt_context_js_namespaceObject = require("./prompt-context.js");
49
51
  const external_task_builder_js_namespaceObject = require("./task-builder.js");
@@ -217,9 +219,9 @@ class TaskExecutor {
217
219
  runner
218
220
  };
219
221
  }
220
- async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
222
+ async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
221
223
  abortSignal?.throwIfAborted();
222
- return (0, external_file_chooser_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
224
+ return (0, external_file_chooser_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions, checkScreenChanges));
223
225
  }
224
226
  invalidateFailedCacheHitLocates(runner, fromIndex) {
225
227
  if (!this.taskCache) return;
@@ -231,7 +233,7 @@ class TaskExecutor {
231
233
  }
232
234
  }
233
235
  }
234
- async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
236
+ async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
235
237
  abortSignal ??= this.getAbortSignal?.();
236
238
  const conversationHistory = new index_js_namespaceObject.ConversationHistory();
237
239
  const promptDisplay = reportOptions?.prompt || (0, external_common_js_namespaceObject.userPromptToString)(userPrompt);
@@ -264,6 +266,8 @@ class TaskExecutor {
264
266
  noIndividualLocateModel
265
267
  });
266
268
  let replanCount = 0;
269
+ let consecutiveScreenChangeReplans = 0;
270
+ const screenChangeReplanLimit = 2;
267
271
  const yamlFlow = [];
268
272
  const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
269
273
  if (void 0 === replanningCycleLimit) throw new Error('replanningCycleLimit is required for TaskExecutor.action');
@@ -394,6 +398,14 @@ class TaskExecutor {
394
398
  } finally{
395
399
  (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callAiEnd');
396
400
  }
401
+ if (this.planningWaitFor && !planResult.delegation) {
402
+ const waitIndex = planResult.actions?.findIndex((action)=>'WaitFor' === action.type) ?? -1;
403
+ if (waitIndex >= 0) {
404
+ planResult.actions = planResult.actions.slice(0, waitIndex + 1);
405
+ planResult.yamlFlow = (0, external_common_js_namespaceObject.buildYamlFlowFromPlans)(planResult.actions, actionSpace);
406
+ planResult.shouldContinuePlanning = true;
407
+ }
408
+ }
397
409
  debug('planResult', JSON.stringify(planResult, null, 2));
398
410
  const { actions, thought, log, memory, error, usage, rawResponse, rawAssistantOutput, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
399
411
  outputString = finalizeMessage;
@@ -456,7 +468,18 @@ class TaskExecutor {
456
468
  });
457
469
  const planResult = result?.output;
458
470
  latestPlanResult = planResult;
459
- if (!planResult?.delegation) {
471
+ let freshness;
472
+ if (checkScreenChanges && !planResult?.delegation) try {
473
+ freshness = await (0, external_plan_freshness_js_namespaceObject.evaluatePlanFreshness)(runner.tasks[runner.tasks.length - 1], planResult?.actions || [], runner.id, planIndex);
474
+ } catch (error) {
475
+ return appendFailedPlan(`Failed to check screen freshness: ${error instanceof Error ? error.message : String(error)}`, planIndex);
476
+ }
477
+ if (freshness?.changed) {
478
+ if (consecutiveScreenChangeReplans >= screenChangeReplanLimit) return appendFailedPlan(`Task stopped: the screen kept changing during planning after ${screenChangeReplanLimit} consecutive screen-change replans. No action from the rejected plans was executed. Wait for the page to settle before trying again.`, planIndex);
479
+ ++consecutiveScreenChangeReplans;
480
+ this.setPendingFeedbackMessage(conversationHistory, await this.getTimeString(), 'The planned action was NOT executed: the screen changed while planning. Inspect the current screenshot and plan again; do not assume the previous action happened.');
481
+ }
482
+ if (!planResult?.delegation && !freshness?.changed) {
460
483
  const plans = planResult?.actions || [];
461
484
  yamlFlow.push(...planResult?.yamlFlow || []);
462
485
  let executables;
@@ -476,9 +499,10 @@ class TaskExecutor {
476
499
  activeActionReporter = (0, external_progress_index_js_namespaceObject.createAiActActionReporter)(planIndex, replanningCycleLimit, (phase, data)=>this.emitAiActProgress(phase, data));
477
500
  try {
478
501
  await session.appendAndRun(executables.tasks);
502
+ if (executables.tasks.some((task)=>'Action Space' === task.type)) consecutiveScreenChangeReplans = 0;
479
503
  this.setPendingFeedbackMessage(conversationHistory, initialTimeString, this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)));
480
504
  } catch (error) {
481
- if ((0, external_action_readiness_js_namespaceObject.isActionReadinessError)(error)) throw error;
505
+ if ((0, external_action_readiness_js_namespaceObject.isActionReadinessError)(error) || (0, external_planned_wait_js_namespaceObject.isPlannedWaitError)(error)) throw error;
482
506
  errorCountInOnePlanningLoop++;
483
507
  const timeString = await this.getTimeString();
484
508
  this.setPendingFeedbackMessage(conversationHistory, timeString, `Error executing running tasks: ${error?.message || String(error)}`);
@@ -632,64 +656,84 @@ class TaskExecutor {
632
656
  };
633
657
  }
634
658
  async waitFor(assertion, opt, modelRuntime) {
635
- const abortSignal = this.getAbortSignal?.();
636
- const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject.parsePrompt)(assertion);
637
- const description = `waitFor: ${textPrompt}`;
638
- const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)('WaitFor', description), {
639
- referenceImages: multimodalPrompt?.images,
640
- abortSignal
641
- });
642
- const runner = session.getRunner();
643
- const executionModelRuntime = {
644
- ...modelRuntime,
645
- executionId: runner.id
646
- };
647
- const { timeoutMs, checkIntervalMs, domIncluded, screenshotIncluded, ...restOpt } = opt;
648
- const serviceExtractOpt = {
649
- domIncluded,
650
- screenshotIncluded,
651
- ...restOpt
652
- };
653
- (0, utils_namespaceObject.assert)(assertion, 'No assertion for waitFor');
654
- (0, utils_namespaceObject.assert)(timeoutMs, 'No timeoutMs for waitFor');
655
- (0, utils_namespaceObject.assert)(checkIntervalMs, 'No checkIntervalMs for waitFor');
656
- (0, utils_namespaceObject.assert)(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
657
- const overallStartTime = Date.now();
658
- let lastCheckStart = overallStartTime;
659
- let errorThought = '';
660
- while(lastCheckStart - overallStartTime <= timeoutMs){
661
- const currentCheckStart = Date.now();
662
- lastCheckStart = currentCheckStart;
663
- abortSignal?.throwIfAborted();
664
- const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
659
+ const timeoutMs = opt.timeoutMs;
660
+ const checkIntervalMs = opt.checkIntervalMs;
661
+ (0, utils_namespaceObject.assert)('number' == typeof timeoutMs && Number.isFinite(timeoutMs) && timeoutMs > 0 && timeoutMs <= 2147483647, 'waitFor timeoutMs must be a finite positive timer duration');
662
+ (0, utils_namespaceObject.assert)('number' == typeof checkIntervalMs && Number.isFinite(checkIntervalMs) && checkIntervalMs > 0 && checkIntervalMs <= timeoutMs, 'waitFor checkIntervalMs must be positive and no greater than timeoutMs');
663
+ const signals = [
664
+ opt.abortSignal,
665
+ this.getAbortSignal?.()
666
+ ].filter((signal)=>void 0 !== signal);
667
+ const callerSignal = signals.length ? AbortSignal.any(signals) : void 0;
668
+ callerSignal?.throwIfAborted();
669
+ const deadline = new AbortController();
670
+ const deadlineAt = Date.now() + timeoutMs;
671
+ const timer = setTimeout(()=>deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`)), timeoutMs);
672
+ const abortSignal = callerSignal ? AbortSignal.any([
673
+ callerSignal,
674
+ deadline.signal
675
+ ]) : deadline.signal;
676
+ try {
677
+ const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject.parsePrompt)(assertion);
678
+ const description = `waitFor: ${textPrompt}`;
679
+ const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)('WaitFor', description), {
680
+ referenceImages: multimodalPrompt?.images,
665
681
  abortSignal
666
682
  });
667
- const result = await session.appendAndRun(queryTask);
668
- if (result?.output) return {
669
- output: void 0,
670
- runner
683
+ const runner = session.getRunner();
684
+ const executionModelRuntime = {
685
+ ...modelRuntime,
686
+ executionId: runner.id
671
687
  };
672
- errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
673
- const now = Date.now();
674
- if (now - currentCheckStart < checkIntervalMs) {
675
- const elapsed = now - currentCheckStart;
676
- const timeRemaining = checkIntervalMs - elapsed;
677
- const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
678
- const { tasks: sleepTasks } = await this.convertPlanToExecutable([
679
- {
680
- type: 'Sleep',
681
- param: {
682
- timeMs: timeRemaining
683
- },
684
- thought
685
- }
686
- ], modelRuntime, modelRuntime, {
688
+ const { timeoutMs: _timeoutMs, checkIntervalMs: _checkIntervalMs, abortSignal: _abortSignal, domIncluded, screenshotIncluded, ...restOpt } = opt;
689
+ const serviceExtractOpt = {
690
+ domIncluded,
691
+ screenshotIncluded,
692
+ ...restOpt
693
+ };
694
+ (0, utils_namespaceObject.assert)(assertion, 'No assertion for waitFor');
695
+ const overallStartTime = Date.now();
696
+ let errorThought = '';
697
+ while(Date.now() - overallStartTime < timeoutMs){
698
+ const currentCheckStart = Date.now();
699
+ abortSignal?.throwIfAborted();
700
+ const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
687
701
  abortSignal
688
702
  });
689
- if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
703
+ const result = await session.appendAndRun(queryTask);
704
+ abortSignal.throwIfAborted();
705
+ if (Date.now() >= deadlineAt) {
706
+ deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`));
707
+ abortSignal.throwIfAborted();
708
+ }
709
+ if (result?.output) return {
710
+ output: void 0,
711
+ runner
712
+ };
713
+ errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
714
+ const now = Date.now();
715
+ if (now - currentCheckStart < checkIntervalMs) {
716
+ const elapsed = now - currentCheckStart;
717
+ const timeRemaining = Math.max(0, Math.min(checkIntervalMs - elapsed, timeoutMs - (Date.now() - overallStartTime)));
718
+ const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
719
+ const { tasks: sleepTasks } = await this.convertPlanToExecutable([
720
+ {
721
+ type: 'Sleep',
722
+ param: {
723
+ timeMs: timeRemaining
724
+ },
725
+ thought
726
+ }
727
+ ], modelRuntime, modelRuntime, {
728
+ abortSignal
729
+ });
730
+ if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
731
+ }
690
732
  }
733
+ return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
734
+ } finally{
735
+ clearTimeout(timer);
691
736
  }
692
- return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
693
737
  }
694
738
  constructor(interfaceInstance, service, opts){
695
739
  _define_property(this, "interface", void 0);
@@ -697,6 +741,7 @@ class TaskExecutor {
697
741
  _define_property(this, "taskCache", void 0);
698
742
  _define_property(this, "getDefaultModelRuntime", void 0);
699
743
  _define_property(this, "providedActionSpace", void 0);
744
+ _define_property(this, "planningWaitFor", void 0);
700
745
  _define_property(this, "taskBuilder", void 0);
701
746
  _define_property(this, "onTaskStartCallback", void 0);
702
747
  _define_property(this, "hooks", void 0);
@@ -710,6 +755,7 @@ class TaskExecutor {
710
755
  this.taskCache = opts.taskCache;
711
756
  this.onTaskStartCallback = opts?.onTaskStart;
712
757
  this.replanningCycleLimit = opts.replanningCycleLimit;
758
+ this.planningWaitFor = opts.planningWaitFor ?? false;
713
759
  this.waitAfterAction = opts.waitAfterAction;
714
760
  this.useDeviceTime = opts.useDeviceTime;
715
761
  this.hooks = opts.hooks;