@midscene/core 1.17.4 → 1.17.5-beta-20261102094111.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/es/agent/agent.mjs +20 -4
  2. package/dist/es/agent/agent.mjs.map +1 -1
  3. package/dist/es/agent/plan-freshness.mjs +39 -0
  4. package/dist/es/agent/plan-freshness.mjs.map +1 -0
  5. package/dist/es/agent/planned-wait.mjs +62 -0
  6. package/dist/es/agent/planned-wait.mjs.map +1 -0
  7. package/dist/es/agent/tasks.mjs +103 -57
  8. package/dist/es/agent/tasks.mjs.map +1 -1
  9. package/dist/es/agent/utils.mjs +1 -1
  10. package/dist/es/ai-model/workflows/planning/custom-planning.mjs +1 -1
  11. package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
  12. package/dist/es/ai-model/workflows/planning/standard-planning.mjs +1 -1
  13. package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
  14. package/dist/es/report-html-template.mjs +1 -1
  15. package/dist/es/task-runner.mjs +16 -1
  16. package/dist/es/task-runner.mjs.map +1 -1
  17. package/dist/es/types.mjs.map +1 -1
  18. package/dist/es/utils.mjs +1 -1
  19. package/dist/lib/agent/agent.js +20 -4
  20. package/dist/lib/agent/agent.js.map +1 -1
  21. package/dist/lib/agent/plan-freshness.js +76 -0
  22. package/dist/lib/agent/plan-freshness.js.map +1 -0
  23. package/dist/lib/agent/planned-wait.js +105 -0
  24. package/dist/lib/agent/planned-wait.js.map +1 -0
  25. package/dist/lib/agent/tasks.js +102 -56
  26. package/dist/lib/agent/tasks.js.map +1 -1
  27. package/dist/lib/agent/utils.js +1 -1
  28. package/dist/lib/ai-model/workflows/planning/custom-planning.js +1 -1
  29. package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
  30. package/dist/lib/ai-model/workflows/planning/standard-planning.js +1 -1
  31. package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
  32. package/dist/lib/report-html-template.js +1 -1
  33. package/dist/lib/task-runner.js +15 -0
  34. package/dist/lib/task-runner.js.map +1 -1
  35. package/dist/lib/types.js.map +1 -1
  36. package/dist/lib/utils.js +1 -1
  37. package/dist/types/agent/agent.d.ts +9 -0
  38. package/dist/types/agent/agent.d.ts.map +1 -1
  39. package/dist/types/agent/plan-freshness.d.ts +15 -0
  40. package/dist/types/agent/plan-freshness.d.ts.map +1 -0
  41. package/dist/types/agent/planned-wait.d.ts +11 -0
  42. package/dist/types/agent/planned-wait.d.ts.map +1 -0
  43. package/dist/types/agent/tasks.d.ts +3 -1
  44. package/dist/types/agent/tasks.d.ts.map +1 -1
  45. package/dist/types/task-runner.d.ts +1 -0
  46. package/dist/types/task-runner.d.ts.map +1 -1
  47. package/dist/types/types.d.ts +10 -0
  48. package/dist/types/types.d.ts.map +1 -1
  49. package/package.json +2 -2
@@ -0,0 +1,62 @@
1
+ import { defineAction } from "../device/index.mjs";
2
+ import { z } from "zod";
3
+ function _define_property(obj, key, value) {
4
+ if (key in obj) Object.defineProperty(obj, key, {
5
+ value: value,
6
+ enumerable: true,
7
+ configurable: true,
8
+ writable: true
9
+ });
10
+ else obj[key] = value;
11
+ return obj;
12
+ }
13
+ const plannedWaitContext = `Conditional waiting policy for all UI tasks:
14
+ - PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as "the interface may take time to respond". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.
15
+ - If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.
16
+ - Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.
17
+ - Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.
18
+ - Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.
19
+ - Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.
20
+ - An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.
21
+ - WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.
22
+ - Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;
23
+ class PlannedWaitError extends Error {
24
+ constructor(cause){
25
+ super(`Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`, {
26
+ cause
27
+ }), _define_property(this, "code", 'PLANNED_WAIT_FAILED');
28
+ this.name = 'PlannedWaitError';
29
+ }
30
+ }
31
+ function isPlannedWaitError(error) {
32
+ const seen = new Set();
33
+ while(error && 'object' == typeof error && !seen.has(error)){
34
+ seen.add(error);
35
+ const value = error;
36
+ if ('PLANNED_WAIT_FAILED' === value.code) return true;
37
+ error = value.cause;
38
+ }
39
+ return false;
40
+ }
41
+ function definePlannedWait(wait) {
42
+ return defineAction({
43
+ name: 'WaitFor',
44
+ description: 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',
45
+ paramSchema: z.object({
46
+ condition: z.string().trim().min(1).describe('The visible condition needed before continuing, which can become true without another user action.')
47
+ }),
48
+ delayBeforeRunner: 0,
49
+ delayAfterRunner: 0,
50
+ call: async ({ condition }, context)=>{
51
+ try {
52
+ await wait(condition, context?.signal);
53
+ if (context) context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;
54
+ } catch (error) {
55
+ throw new PlannedWaitError(error);
56
+ }
57
+ }
58
+ });
59
+ }
60
+ export { PlannedWaitError, definePlannedWait, isPlannedWaitError, plannedWaitContext };
61
+
62
+ //# sourceMappingURL=planned-wait.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agent/planned-wait.mjs","sources":["../../../src/agent/planned-wait.ts"],"sourcesContent":["import { defineAction } from '@/device';\nimport { z } from 'zod';\n\n/** Planning guidance is only attached when conditional waiting is enabled. */\nexport const plannedWaitContext = `Conditional waiting policy for all UI tasks:\n- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as \"the interface may take time to respond\". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.\n- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.\n- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.\n- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.\n- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.\n- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.\n- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.\n- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.\n- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;\n\nexport class PlannedWaitError extends Error {\n readonly code = 'PLANNED_WAIT_FAILED';\n constructor(cause: unknown) {\n super(\n `Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`,\n { cause },\n );\n this.name = 'PlannedWaitError';\n }\n}\n\nexport function isPlannedWaitError(error: unknown): boolean {\n const seen = new Set<unknown>();\n while (error && typeof error === 'object' && !seen.has(error)) {\n seen.add(error);\n const value = error as { code?: unknown; cause?: unknown };\n if (value.code === 'PLANNED_WAIT_FAILED') return true;\n error = value.cause;\n }\n return false;\n}\n\nexport function definePlannedWait(\n wait: (condition: string, signal?: AbortSignal) => Promise<void>,\n) {\n return defineAction({\n name: 'WaitFor',\n description:\n 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',\n paramSchema: z.object({\n condition: z\n .string()\n .trim()\n .min(1)\n .describe(\n 'The visible condition needed before continuing, which can become true without another user action.',\n ),\n }),\n delayBeforeRunner: 0,\n delayAfterRunner: 0,\n call: async ({ condition }, context) => {\n try {\n await wait(condition, context?.signal);\n if (context) {\n context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;\n }\n } catch (error) {\n throw new PlannedWaitError(error);\n }\n },\n });\n}\n"],"names":["plannedWaitContext","PlannedWaitError","Error","cause","String","isPlannedWaitError","error","seen","Set","value","definePlannedWait","wait","defineAction","z","condition","context"],"mappings":";;;;;;;;;;;;AAIO,MAAMA,qBAAqB,CAAC;;;;;;;;;6JAS0H,CAAC;AAEvJ,MAAMC,yBAAyBC;IAEpC,YAAYC,KAAc,CAAE;QAC1B,KAAK,CACH,CAAC,wBAAwB,EAAEA,iBAAiBD,QAAQC,MAAM,OAAO,GAAGC,OAAOD,QAAQ,EACnF;YAAEA;QAAM,IAJZ,uBAAS,QAAO;QAMd,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEO,SAASE,mBAAmBC,KAAc;IAC/C,MAAMC,OAAO,IAAIC;IACjB,MAAOF,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,KAAK,GAAG,CAACD,OAAQ;QAC7DC,KAAK,GAAG,CAACD;QACT,MAAMG,QAAQH;QACd,IAAIG,AAAe,0BAAfA,MAAM,IAAI,EAA4B,OAAO;QACjDH,QAAQG,MAAM,KAAK;IACrB;IACA,OAAO;AACT;AAEO,SAASC,kBACdC,IAAgE;IAEhE,OAAOC,aAAa;QAClB,MAAM;QACN,aACE;QACF,aAAaC,EAAE,MAAM,CAAC;YACpB,WAAWA,EAAAA,MACF,GACN,IAAI,GACJ,GAAG,CAAC,GACJ,QAAQ,CACP;QAEN;QACA,mBAAmB;QACnB,kBAAkB;QAClB,MAAM,OAAO,EAAEC,SAAS,EAAE,EAAEC;YAC1B,IAAI;gBACF,MAAMJ,KAAKG,WAAWC,SAAS;gBAC/B,IAAIA,SACFA,QAAQ,IAAI,CAAC,gBAAgB,GAAG,CAAC,mBAAmB,EAAED,UAAU,2HAA2H,CAAC;YAEhM,EAAE,OAAOR,OAAO;gBACd,MAAM,IAAIL,iBAAiBK;YAC7B;QACF;IACF;AACF"}
@@ -2,7 +2,7 @@ import { AIResponseParseError, ConversationHistory } from "../ai-model/index.mjs
2
2
  import { buildTypeQueryDemandValue } from "../ai-model/prompt/insight/index.mjs";
3
3
  import { prepareUserPrompt } from "../ai-model/shared/multimodal-prompt.mjs";
4
4
  import { standardPlan } from "../ai-model/workflows/planning/index.mjs";
5
- import { getReadableTimeString, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
5
+ import { buildYamlFlowFromPlans, getReadableTimeString, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
6
6
  import { defineActionSleep } from "../device/index.mjs";
7
7
  import { TaskExecutionError } from "../task-runner.mjs";
8
8
  import { ServiceError, aiActProgressScope } from "../types.mjs";
@@ -12,6 +12,8 @@ import { resolveSystemOneEndpoint } from "../ai-model/service-caller/system-one/
12
12
  import { isActionReadinessError } from "./action-readiness.mjs";
13
13
  import { ExecutionSession } from "./execution-session.mjs";
14
14
  import { withFileChooser } from "./file-chooser.mjs";
15
+ import { evaluatePlanFreshness } from "./plan-freshness.mjs";
16
+ import { isPlannedWaitError } from "./planned-wait.mjs";
15
17
  import { createAiActActionReporter, errorMessageForAiAct } from "./progress/index.mjs";
16
18
  import { renderAIContext } from "./prompt-context.mjs";
17
19
  import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
@@ -185,9 +187,9 @@ class TaskExecutor {
185
187
  runner
186
188
  };
187
189
  }
188
- async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
190
+ async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
189
191
  abortSignal?.throwIfAborted();
190
- return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
192
+ return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions, checkScreenChanges));
191
193
  }
192
194
  invalidateFailedCacheHitLocates(runner, fromIndex) {
193
195
  if (!this.taskCache) return;
@@ -199,7 +201,7 @@ class TaskExecutor {
199
201
  }
200
202
  }
201
203
  }
202
- async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
204
+ async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
203
205
  abortSignal ??= this.getAbortSignal?.();
204
206
  const conversationHistory = new ConversationHistory();
205
207
  const promptDisplay = reportOptions?.prompt || userPromptToString(userPrompt);
@@ -232,6 +234,8 @@ class TaskExecutor {
232
234
  noIndividualLocateModel
233
235
  });
234
236
  let replanCount = 0;
237
+ let consecutiveScreenChangeReplans = 0;
238
+ const screenChangeReplanLimit = 2;
235
239
  const yamlFlow = [];
236
240
  const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
237
241
  if (void 0 === replanningCycleLimit) throw new Error('replanningCycleLimit is required for TaskExecutor.action');
@@ -362,6 +366,14 @@ class TaskExecutor {
362
366
  } finally{
363
367
  setTimingFieldOnce(timing, 'callAiEnd');
364
368
  }
369
+ if (this.planningWaitFor && !planResult.delegation) {
370
+ const waitIndex = planResult.actions?.findIndex((action)=>'WaitFor' === action.type) ?? -1;
371
+ if (waitIndex >= 0) {
372
+ planResult.actions = planResult.actions.slice(0, waitIndex + 1);
373
+ planResult.yamlFlow = buildYamlFlowFromPlans(planResult.actions, actionSpace);
374
+ planResult.shouldContinuePlanning = true;
375
+ }
376
+ }
365
377
  debug('planResult', JSON.stringify(planResult, null, 2));
366
378
  const { actions, thought, log, memory, error, usage, rawResponse, rawAssistantOutput, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
367
379
  outputString = finalizeMessage;
@@ -424,7 +436,18 @@ class TaskExecutor {
424
436
  });
425
437
  const planResult = result?.output;
426
438
  latestPlanResult = planResult;
427
- if (!planResult?.delegation) {
439
+ let freshness;
440
+ if (checkScreenChanges && !planResult?.delegation) try {
441
+ freshness = await evaluatePlanFreshness(runner.tasks[runner.tasks.length - 1], planResult?.actions || [], runner.id, planIndex);
442
+ } catch (error) {
443
+ return appendFailedPlan(`Failed to check screen freshness: ${error instanceof Error ? error.message : String(error)}`, planIndex);
444
+ }
445
+ if (freshness?.changed) {
446
+ if (consecutiveScreenChangeReplans >= screenChangeReplanLimit) return appendFailedPlan(`Task stopped: the screen kept changing during planning after ${screenChangeReplanLimit} consecutive screen-change replans. No action from the rejected plans was executed. Wait for the page to settle before trying again.`, planIndex);
447
+ ++consecutiveScreenChangeReplans;
448
+ this.setPendingFeedbackMessage(conversationHistory, await this.getTimeString(), 'The planned action was NOT executed: the screen changed while planning. Inspect the current screenshot and plan again; do not assume the previous action happened.');
449
+ }
450
+ if (!planResult?.delegation && !freshness?.changed) {
428
451
  const plans = planResult?.actions || [];
429
452
  yamlFlow.push(...planResult?.yamlFlow || []);
430
453
  let executables;
@@ -444,9 +467,10 @@ class TaskExecutor {
444
467
  activeActionReporter = createAiActActionReporter(planIndex, replanningCycleLimit, (phase, data)=>this.emitAiActProgress(phase, data));
445
468
  try {
446
469
  await session.appendAndRun(executables.tasks);
470
+ if (executables.tasks.some((task)=>'Action Space' === task.type)) consecutiveScreenChangeReplans = 0;
447
471
  this.setPendingFeedbackMessage(conversationHistory, initialTimeString, this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)));
448
472
  } catch (error) {
449
- if (isActionReadinessError(error)) throw error;
473
+ if (isActionReadinessError(error) || isPlannedWaitError(error)) throw error;
450
474
  errorCountInOnePlanningLoop++;
451
475
  const timeString = await this.getTimeString();
452
476
  this.setPendingFeedbackMessage(conversationHistory, timeString, `Error executing running tasks: ${error?.message || String(error)}`);
@@ -600,64 +624,84 @@ class TaskExecutor {
600
624
  };
601
625
  }
602
626
  async waitFor(assertion, opt, modelRuntime) {
603
- const abortSignal = this.getAbortSignal?.();
604
- const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
605
- const description = `waitFor: ${textPrompt}`;
606
- const session = this.createExecutionSession(taskTitleStr('WaitFor', description), {
607
- referenceImages: multimodalPrompt?.images,
608
- abortSignal
609
- });
610
- const runner = session.getRunner();
611
- const executionModelRuntime = {
612
- ...modelRuntime,
613
- executionId: runner.id
614
- };
615
- const { timeoutMs, checkIntervalMs, domIncluded, screenshotIncluded, ...restOpt } = opt;
616
- const serviceExtractOpt = {
617
- domIncluded,
618
- screenshotIncluded,
619
- ...restOpt
620
- };
621
- assert(assertion, 'No assertion for waitFor');
622
- assert(timeoutMs, 'No timeoutMs for waitFor');
623
- assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
624
- assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
625
- const overallStartTime = Date.now();
626
- let lastCheckStart = overallStartTime;
627
- let errorThought = '';
628
- while(lastCheckStart - overallStartTime <= timeoutMs){
629
- const currentCheckStart = Date.now();
630
- lastCheckStart = currentCheckStart;
631
- abortSignal?.throwIfAborted();
632
- const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
627
+ const timeoutMs = opt.timeoutMs;
628
+ const checkIntervalMs = opt.checkIntervalMs;
629
+ assert('number' == typeof timeoutMs && Number.isFinite(timeoutMs) && timeoutMs > 0 && timeoutMs <= 2147483647, 'waitFor timeoutMs must be a finite positive timer duration');
630
+ assert('number' == typeof checkIntervalMs && Number.isFinite(checkIntervalMs) && checkIntervalMs > 0 && checkIntervalMs <= timeoutMs, 'waitFor checkIntervalMs must be positive and no greater than timeoutMs');
631
+ const signals = [
632
+ opt.abortSignal,
633
+ this.getAbortSignal?.()
634
+ ].filter((signal)=>void 0 !== signal);
635
+ const callerSignal = signals.length ? AbortSignal.any(signals) : void 0;
636
+ callerSignal?.throwIfAborted();
637
+ const deadline = new AbortController();
638
+ const deadlineAt = Date.now() + timeoutMs;
639
+ const timer = setTimeout(()=>deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`)), timeoutMs);
640
+ const abortSignal = callerSignal ? AbortSignal.any([
641
+ callerSignal,
642
+ deadline.signal
643
+ ]) : deadline.signal;
644
+ try {
645
+ const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
646
+ const description = `waitFor: ${textPrompt}`;
647
+ const session = this.createExecutionSession(taskTitleStr('WaitFor', description), {
648
+ referenceImages: multimodalPrompt?.images,
633
649
  abortSignal
634
650
  });
635
- const result = await session.appendAndRun(queryTask);
636
- if (result?.output) return {
637
- output: void 0,
638
- runner
651
+ const runner = session.getRunner();
652
+ const executionModelRuntime = {
653
+ ...modelRuntime,
654
+ executionId: runner.id
639
655
  };
640
- errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
641
- const now = Date.now();
642
- if (now - currentCheckStart < checkIntervalMs) {
643
- const elapsed = now - currentCheckStart;
644
- const timeRemaining = checkIntervalMs - elapsed;
645
- const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
646
- const { tasks: sleepTasks } = await this.convertPlanToExecutable([
647
- {
648
- type: 'Sleep',
649
- param: {
650
- timeMs: timeRemaining
651
- },
652
- thought
653
- }
654
- ], modelRuntime, modelRuntime, {
656
+ const { timeoutMs: _timeoutMs, checkIntervalMs: _checkIntervalMs, abortSignal: _abortSignal, domIncluded, screenshotIncluded, ...restOpt } = opt;
657
+ const serviceExtractOpt = {
658
+ domIncluded,
659
+ screenshotIncluded,
660
+ ...restOpt
661
+ };
662
+ assert(assertion, 'No assertion for waitFor');
663
+ const overallStartTime = Date.now();
664
+ let errorThought = '';
665
+ while(Date.now() - overallStartTime < timeoutMs){
666
+ const currentCheckStart = Date.now();
667
+ abortSignal?.throwIfAborted();
668
+ const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
655
669
  abortSignal
656
670
  });
657
- if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
671
+ const result = await session.appendAndRun(queryTask);
672
+ abortSignal.throwIfAborted();
673
+ if (Date.now() >= deadlineAt) {
674
+ deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`));
675
+ abortSignal.throwIfAborted();
676
+ }
677
+ if (result?.output) return {
678
+ output: void 0,
679
+ runner
680
+ };
681
+ errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
682
+ const now = Date.now();
683
+ if (now - currentCheckStart < checkIntervalMs) {
684
+ const elapsed = now - currentCheckStart;
685
+ const timeRemaining = Math.max(0, Math.min(checkIntervalMs - elapsed, timeoutMs - (Date.now() - overallStartTime)));
686
+ const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
687
+ const { tasks: sleepTasks } = await this.convertPlanToExecutable([
688
+ {
689
+ type: 'Sleep',
690
+ param: {
691
+ timeMs: timeRemaining
692
+ },
693
+ thought
694
+ }
695
+ ], modelRuntime, modelRuntime, {
696
+ abortSignal
697
+ });
698
+ if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
699
+ }
658
700
  }
701
+ return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
702
+ } finally{
703
+ clearTimeout(timer);
659
704
  }
660
- return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
661
705
  }
662
706
  constructor(interfaceInstance, service, opts){
663
707
  _define_property(this, "interface", void 0);
@@ -665,6 +709,7 @@ class TaskExecutor {
665
709
  _define_property(this, "taskCache", void 0);
666
710
  _define_property(this, "getDefaultModelRuntime", void 0);
667
711
  _define_property(this, "providedActionSpace", void 0);
712
+ _define_property(this, "planningWaitFor", void 0);
668
713
  _define_property(this, "taskBuilder", void 0);
669
714
  _define_property(this, "onTaskStartCallback", void 0);
670
715
  _define_property(this, "hooks", void 0);
@@ -678,6 +723,7 @@ class TaskExecutor {
678
723
  this.taskCache = opts.taskCache;
679
724
  this.onTaskStartCallback = opts?.onTaskStart;
680
725
  this.replanningCycleLimit = opts.replanningCycleLimit;
726
+ this.planningWaitFor = opts.planningWaitFor ?? false;
681
727
  this.waitAfterAction = opts.waitAfterAction;
682
728
  this.useDeviceTime = opts.useDeviceTime;
683
729
  this.hooks = opts.hooks;