@midscene/core 1.17.4 → 1.17.5-beta-20261102094111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +20 -4
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/plan-freshness.mjs +39 -0
- package/dist/es/agent/plan-freshness.mjs.map +1 -0
- package/dist/es/agent/planned-wait.mjs +62 -0
- package/dist/es/agent/planned-wait.mjs.map +1 -0
- package/dist/es/agent/tasks.mjs +103 -57
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs +1 -1
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
- package/dist/es/report-html-template.mjs +1 -1
- package/dist/es/task-runner.mjs +16 -1
- package/dist/es/task-runner.mjs.map +1 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +1 -1
- package/dist/lib/agent/agent.js +20 -4
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/plan-freshness.js +76 -0
- package/dist/lib/agent/plan-freshness.js.map +1 -0
- package/dist/lib/agent/planned-wait.js +105 -0
- package/dist/lib/agent/planned-wait.js.map +1 -0
- package/dist/lib/agent/tasks.js +102 -56
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/utils.js +1 -1
- package/dist/lib/ai-model/workflows/planning/custom-planning.js +1 -1
- package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
- package/dist/lib/ai-model/workflows/planning/standard-planning.js +1 -1
- package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
- package/dist/lib/report-html-template.js +1 -1
- package/dist/lib/task-runner.js +15 -0
- package/dist/lib/task-runner.js.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/types/agent/agent.d.ts +9 -0
- package/dist/types/agent/agent.d.ts.map +1 -1
- package/dist/types/agent/plan-freshness.d.ts +15 -0
- package/dist/types/agent/plan-freshness.d.ts.map +1 -0
- package/dist/types/agent/planned-wait.d.ts +11 -0
- package/dist/types/agent/planned-wait.d.ts.map +1 -0
- package/dist/types/agent/tasks.d.ts +3 -1
- package/dist/types/agent/tasks.d.ts.map +1 -1
- package/dist/types/task-runner.d.ts +1 -0
- package/dist/types/task-runner.d.ts.map +1 -1
- package/dist/types/types.d.ts +10 -0
- package/dist/types/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { defineAction } from "../device/index.mjs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
function _define_property(obj, key, value) {
|
|
4
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
5
|
+
value: value,
|
|
6
|
+
enumerable: true,
|
|
7
|
+
configurable: true,
|
|
8
|
+
writable: true
|
|
9
|
+
});
|
|
10
|
+
else obj[key] = value;
|
|
11
|
+
return obj;
|
|
12
|
+
}
|
|
13
|
+
const plannedWaitContext = `Conditional waiting policy for all UI tasks:
|
|
14
|
+
- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as "the interface may take time to respond". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.
|
|
15
|
+
- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.
|
|
16
|
+
- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.
|
|
17
|
+
- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.
|
|
18
|
+
- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.
|
|
19
|
+
- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.
|
|
20
|
+
- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.
|
|
21
|
+
- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.
|
|
22
|
+
- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;
|
|
23
|
+
class PlannedWaitError extends Error {
|
|
24
|
+
constructor(cause){
|
|
25
|
+
super(`Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`, {
|
|
26
|
+
cause
|
|
27
|
+
}), _define_property(this, "code", 'PLANNED_WAIT_FAILED');
|
|
28
|
+
this.name = 'PlannedWaitError';
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function isPlannedWaitError(error) {
|
|
32
|
+
const seen = new Set();
|
|
33
|
+
while(error && 'object' == typeof error && !seen.has(error)){
|
|
34
|
+
seen.add(error);
|
|
35
|
+
const value = error;
|
|
36
|
+
if ('PLANNED_WAIT_FAILED' === value.code) return true;
|
|
37
|
+
error = value.cause;
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
function definePlannedWait(wait) {
|
|
42
|
+
return defineAction({
|
|
43
|
+
name: 'WaitFor',
|
|
44
|
+
description: 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',
|
|
45
|
+
paramSchema: z.object({
|
|
46
|
+
condition: z.string().trim().min(1).describe('The visible condition needed before continuing, which can become true without another user action.')
|
|
47
|
+
}),
|
|
48
|
+
delayBeforeRunner: 0,
|
|
49
|
+
delayAfterRunner: 0,
|
|
50
|
+
call: async ({ condition }, context)=>{
|
|
51
|
+
try {
|
|
52
|
+
await wait(condition, context?.signal);
|
|
53
|
+
if (context) context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;
|
|
54
|
+
} catch (error) {
|
|
55
|
+
throw new PlannedWaitError(error);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
export { PlannedWaitError, definePlannedWait, isPlannedWaitError, plannedWaitContext };
|
|
61
|
+
|
|
62
|
+
//# sourceMappingURL=planned-wait.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent/planned-wait.mjs","sources":["../../../src/agent/planned-wait.ts"],"sourcesContent":["import { defineAction } from '@/device';\nimport { z } from 'zod';\n\n/** Planning guidance is only attached when conditional waiting is enabled. */\nexport const plannedWaitContext = `Conditional waiting policy for all UI tasks:\n- PRIORITY: An explicit current business error, permission denial, read-only account or validation failure overrides generic text such as \"the interface may take time to respond\". Do not reinterpret such an error as loading. If no user-authorized action can resolve it, fail immediately with the visible reason; do NOT WaitFor that prohibition to disappear or a disabled control to enable. Only an explicit indication of a temporary/in-progress recovery justifies waiting despite an error.\n- If the latest screenshot already shows the next target visible and enabled, act immediately. Do NOT insert WaitFor or Sleep as a routine precaution before/after every action, or to recheck a condition that is already visibly satisfied.\n- Before choosing WaitFor, identify a specific unmet prerequisite or a pending result of an already performed action. Explain that evidence briefly in the planning thought. A validation error or missing required user action is not an asynchronous wait.\n- Before an interaction, inspect whether its target is currently visible and enabled. A disabled or loading control is not ready: use WaitFor for that prerequisite BEFORE attempting Tap. Do not test-click disabled controls.\n- Choose the nearest unmet condition that can become true WITHOUT another user action. After entering text, if the next control is disabled, wait for that control to become enabled; do not wait for the final submission result before submitting.\n- Preserve already entered text. Do not add/delete characters, clear/retype, refocus or press Enter just to force asynchronous UI state to update. Input content must only be changed when required by the user's task or an explicit validation error.\n- An action call returning only proves dispatch, not application acceptance. Do not assume a disabled control accepted a click. For a dispatched action with pending UI feedback, wait for its relevant next state instead of replaying it merely because the screen is unchanged.\n- WaitFor only observes. Once its prerequisite is satisfied, execute the pending action. Wait for an outcome only when the required triggering action has been performed on a ready control. Complete only when the user's requested end state is observed; do not expand the task scope.\n- Never use Sleep as a substitute when a concrete visual readiness condition is available. Report explicit errors rather than waiting for impossible success.`;\n\nexport class PlannedWaitError extends Error {\n readonly code = 'PLANNED_WAIT_FAILED';\n constructor(cause: unknown) {\n super(\n `Planned WaitFor failed: ${cause instanceof Error ? cause.message : String(cause)}`,\n { cause },\n );\n this.name = 'PlannedWaitError';\n }\n}\n\nexport function isPlannedWaitError(error: unknown): boolean {\n const seen = new Set<unknown>();\n while (error && typeof error === 'object' && !seen.has(error)) {\n seen.add(error);\n const value = error as { code?: unknown; cause?: unknown };\n if (value.code === 'PLANNED_WAIT_FAILED') return true;\n error = value.cause;\n }\n return false;\n}\n\nexport function definePlannedWait(\n wait: (condition: string, signal?: AbortSignal) => Promise<void>,\n) {\n return defineAction({\n name: 'WaitFor',\n description:\n 'Wait for an unmet visible prerequisite or a pending result of an accepted action. Skip this if the current screenshot already satisfies the condition. Never wait for a result whose triggering action has not happened, or for an explicit business error to disappear. Only observes; does not click or edit. Times out after 30 seconds.',\n paramSchema: z.object({\n condition: z\n .string()\n .trim()\n .min(1)\n .describe(\n 'The visible condition needed before continuing, which can become true without another user action.',\n ),\n }),\n delayBeforeRunner: 0,\n delayAfterRunner: 0,\n call: async ({ condition }, context) => {\n try {\n await wait(condition, context?.signal);\n if (context) {\n context.task.planningFeedback = `WaitFor succeeded: ${condition}. Continue from the current screenshot; do not repeat this wait unless new evidence shows the condition became false again.`;\n }\n } catch (error) {\n throw new PlannedWaitError(error);\n }\n },\n });\n}\n"],"names":["plannedWaitContext","PlannedWaitError","Error","cause","String","isPlannedWaitError","error","seen","Set","value","definePlannedWait","wait","defineAction","z","condition","context"],"mappings":";;;;;;;;;;;;AAIO,MAAMA,qBAAqB,CAAC;;;;;;;;;6JAS0H,CAAC;AAEvJ,MAAMC,yBAAyBC;IAEpC,YAAYC,KAAc,CAAE;QAC1B,KAAK,CACH,CAAC,wBAAwB,EAAEA,iBAAiBD,QAAQC,MAAM,OAAO,GAAGC,OAAOD,QAAQ,EACnF;YAAEA;QAAM,IAJZ,uBAAS,QAAO;QAMd,IAAI,CAAC,IAAI,GAAG;IACd;AACF;AAEO,SAASE,mBAAmBC,KAAc;IAC/C,MAAMC,OAAO,IAAIC;IACjB,MAAOF,SAAS,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,KAAK,GAAG,CAACD,OAAQ;QAC7DC,KAAK,GAAG,CAACD;QACT,MAAMG,QAAQH;QACd,IAAIG,AAAe,0BAAfA,MAAM,IAAI,EAA4B,OAAO;QACjDH,QAAQG,MAAM,KAAK;IACrB;IACA,OAAO;AACT;AAEO,SAASC,kBACdC,IAAgE;IAEhE,OAAOC,aAAa;QAClB,MAAM;QACN,aACE;QACF,aAAaC,EAAE,MAAM,CAAC;YACpB,WAAWA,EAAAA,MACF,GACN,IAAI,GACJ,GAAG,CAAC,GACJ,QAAQ,CACP;QAEN;QACA,mBAAmB;QACnB,kBAAkB;QAClB,MAAM,OAAO,EAAEC,SAAS,EAAE,EAAEC;YAC1B,IAAI;gBACF,MAAMJ,KAAKG,WAAWC,SAAS;gBAC/B,IAAIA,SACFA,QAAQ,IAAI,CAAC,gBAAgB,GAAG,CAAC,mBAAmB,EAAED,UAAU,2HAA2H,CAAC;YAEhM,EAAE,OAAOR,OAAO;gBACd,MAAM,IAAIL,iBAAiBK;YAC7B;QACF;IACF;AACF"}
|
package/dist/es/agent/tasks.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { AIResponseParseError, ConversationHistory } from "../ai-model/index.mjs
|
|
|
2
2
|
import { buildTypeQueryDemandValue } from "../ai-model/prompt/insight/index.mjs";
|
|
3
3
|
import { prepareUserPrompt } from "../ai-model/shared/multimodal-prompt.mjs";
|
|
4
4
|
import { standardPlan } from "../ai-model/workflows/planning/index.mjs";
|
|
5
|
-
import { getReadableTimeString, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
|
|
5
|
+
import { buildYamlFlowFromPlans, getReadableTimeString, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
|
|
6
6
|
import { defineActionSleep } from "../device/index.mjs";
|
|
7
7
|
import { TaskExecutionError } from "../task-runner.mjs";
|
|
8
8
|
import { ServiceError, aiActProgressScope } from "../types.mjs";
|
|
@@ -12,6 +12,8 @@ import { resolveSystemOneEndpoint } from "../ai-model/service-caller/system-one/
|
|
|
12
12
|
import { isActionReadinessError } from "./action-readiness.mjs";
|
|
13
13
|
import { ExecutionSession } from "./execution-session.mjs";
|
|
14
14
|
import { withFileChooser } from "./file-chooser.mjs";
|
|
15
|
+
import { evaluatePlanFreshness } from "./plan-freshness.mjs";
|
|
16
|
+
import { isPlannedWaitError } from "./planned-wait.mjs";
|
|
15
17
|
import { createAiActActionReporter, errorMessageForAiAct } from "./progress/index.mjs";
|
|
16
18
|
import { renderAIContext } from "./prompt-context.mjs";
|
|
17
19
|
import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
|
|
@@ -185,9 +187,9 @@ class TaskExecutor {
|
|
|
185
187
|
runner
|
|
186
188
|
};
|
|
187
189
|
}
|
|
188
|
-
async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
|
|
190
|
+
async action(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
|
|
189
191
|
abortSignal?.throwIfAborted();
|
|
190
|
-
return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
|
|
192
|
+
return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions, checkScreenChanges));
|
|
191
193
|
}
|
|
192
194
|
invalidateFailedCacheHitLocates(runner, fromIndex) {
|
|
193
195
|
if (!this.taskCache) return;
|
|
@@ -199,7 +201,7 @@ class TaskExecutor {
|
|
|
199
201
|
}
|
|
200
202
|
}
|
|
201
203
|
}
|
|
202
|
-
async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
|
|
204
|
+
async runAction(userPrompt, planningModel, locateModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions, checkScreenChanges = false) {
|
|
203
205
|
abortSignal ??= this.getAbortSignal?.();
|
|
204
206
|
const conversationHistory = new ConversationHistory();
|
|
205
207
|
const promptDisplay = reportOptions?.prompt || userPromptToString(userPrompt);
|
|
@@ -232,6 +234,8 @@ class TaskExecutor {
|
|
|
232
234
|
noIndividualLocateModel
|
|
233
235
|
});
|
|
234
236
|
let replanCount = 0;
|
|
237
|
+
let consecutiveScreenChangeReplans = 0;
|
|
238
|
+
const screenChangeReplanLimit = 2;
|
|
235
239
|
const yamlFlow = [];
|
|
236
240
|
const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
|
|
237
241
|
if (void 0 === replanningCycleLimit) throw new Error('replanningCycleLimit is required for TaskExecutor.action');
|
|
@@ -362,6 +366,14 @@ class TaskExecutor {
|
|
|
362
366
|
} finally{
|
|
363
367
|
setTimingFieldOnce(timing, 'callAiEnd');
|
|
364
368
|
}
|
|
369
|
+
if (this.planningWaitFor && !planResult.delegation) {
|
|
370
|
+
const waitIndex = planResult.actions?.findIndex((action)=>'WaitFor' === action.type) ?? -1;
|
|
371
|
+
if (waitIndex >= 0) {
|
|
372
|
+
planResult.actions = planResult.actions.slice(0, waitIndex + 1);
|
|
373
|
+
planResult.yamlFlow = buildYamlFlowFromPlans(planResult.actions, actionSpace);
|
|
374
|
+
planResult.shouldContinuePlanning = true;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
365
377
|
debug('planResult', JSON.stringify(planResult, null, 2));
|
|
366
378
|
const { actions, thought, log, memory, error, usage, rawResponse, rawAssistantOutput, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
|
|
367
379
|
outputString = finalizeMessage;
|
|
@@ -424,7 +436,18 @@ class TaskExecutor {
|
|
|
424
436
|
});
|
|
425
437
|
const planResult = result?.output;
|
|
426
438
|
latestPlanResult = planResult;
|
|
427
|
-
|
|
439
|
+
let freshness;
|
|
440
|
+
if (checkScreenChanges && !planResult?.delegation) try {
|
|
441
|
+
freshness = await evaluatePlanFreshness(runner.tasks[runner.tasks.length - 1], planResult?.actions || [], runner.id, planIndex);
|
|
442
|
+
} catch (error) {
|
|
443
|
+
return appendFailedPlan(`Failed to check screen freshness: ${error instanceof Error ? error.message : String(error)}`, planIndex);
|
|
444
|
+
}
|
|
445
|
+
if (freshness?.changed) {
|
|
446
|
+
if (consecutiveScreenChangeReplans >= screenChangeReplanLimit) return appendFailedPlan(`Task stopped: the screen kept changing during planning after ${screenChangeReplanLimit} consecutive screen-change replans. No action from the rejected plans was executed. Wait for the page to settle before trying again.`, planIndex);
|
|
447
|
+
++consecutiveScreenChangeReplans;
|
|
448
|
+
this.setPendingFeedbackMessage(conversationHistory, await this.getTimeString(), 'The planned action was NOT executed: the screen changed while planning. Inspect the current screenshot and plan again; do not assume the previous action happened.');
|
|
449
|
+
}
|
|
450
|
+
if (!planResult?.delegation && !freshness?.changed) {
|
|
428
451
|
const plans = planResult?.actions || [];
|
|
429
452
|
yamlFlow.push(...planResult?.yamlFlow || []);
|
|
430
453
|
let executables;
|
|
@@ -444,9 +467,10 @@ class TaskExecutor {
|
|
|
444
467
|
activeActionReporter = createAiActActionReporter(planIndex, replanningCycleLimit, (phase, data)=>this.emitAiActProgress(phase, data));
|
|
445
468
|
try {
|
|
446
469
|
await session.appendAndRun(executables.tasks);
|
|
470
|
+
if (executables.tasks.some((task)=>'Action Space' === task.type)) consecutiveScreenChangeReplans = 0;
|
|
447
471
|
this.setPendingFeedbackMessage(conversationHistory, initialTimeString, this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)));
|
|
448
472
|
} catch (error) {
|
|
449
|
-
if (isActionReadinessError(error)) throw error;
|
|
473
|
+
if (isActionReadinessError(error) || isPlannedWaitError(error)) throw error;
|
|
450
474
|
errorCountInOnePlanningLoop++;
|
|
451
475
|
const timeString = await this.getTimeString();
|
|
452
476
|
this.setPendingFeedbackMessage(conversationHistory, timeString, `Error executing running tasks: ${error?.message || String(error)}`);
|
|
@@ -600,64 +624,84 @@ class TaskExecutor {
|
|
|
600
624
|
};
|
|
601
625
|
}
|
|
602
626
|
async waitFor(assertion, opt, modelRuntime) {
|
|
603
|
-
const
|
|
604
|
-
const
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
abortSignal
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
const
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
const
|
|
616
|
-
const
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
const overallStartTime = Date.now();
|
|
626
|
-
let lastCheckStart = overallStartTime;
|
|
627
|
-
let errorThought = '';
|
|
628
|
-
while(lastCheckStart - overallStartTime <= timeoutMs){
|
|
629
|
-
const currentCheckStart = Date.now();
|
|
630
|
-
lastCheckStart = currentCheckStart;
|
|
631
|
-
abortSignal?.throwIfAborted();
|
|
632
|
-
const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
|
|
627
|
+
const timeoutMs = opt.timeoutMs;
|
|
628
|
+
const checkIntervalMs = opt.checkIntervalMs;
|
|
629
|
+
assert('number' == typeof timeoutMs && Number.isFinite(timeoutMs) && timeoutMs > 0 && timeoutMs <= 2147483647, 'waitFor timeoutMs must be a finite positive timer duration');
|
|
630
|
+
assert('number' == typeof checkIntervalMs && Number.isFinite(checkIntervalMs) && checkIntervalMs > 0 && checkIntervalMs <= timeoutMs, 'waitFor checkIntervalMs must be positive and no greater than timeoutMs');
|
|
631
|
+
const signals = [
|
|
632
|
+
opt.abortSignal,
|
|
633
|
+
this.getAbortSignal?.()
|
|
634
|
+
].filter((signal)=>void 0 !== signal);
|
|
635
|
+
const callerSignal = signals.length ? AbortSignal.any(signals) : void 0;
|
|
636
|
+
callerSignal?.throwIfAborted();
|
|
637
|
+
const deadline = new AbortController();
|
|
638
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
639
|
+
const timer = setTimeout(()=>deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`)), timeoutMs);
|
|
640
|
+
const abortSignal = callerSignal ? AbortSignal.any([
|
|
641
|
+
callerSignal,
|
|
642
|
+
deadline.signal
|
|
643
|
+
]) : deadline.signal;
|
|
644
|
+
try {
|
|
645
|
+
const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
|
|
646
|
+
const description = `waitFor: ${textPrompt}`;
|
|
647
|
+
const session = this.createExecutionSession(taskTitleStr('WaitFor', description), {
|
|
648
|
+
referenceImages: multimodalPrompt?.images,
|
|
633
649
|
abortSignal
|
|
634
650
|
});
|
|
635
|
-
const
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
runner
|
|
651
|
+
const runner = session.getRunner();
|
|
652
|
+
const executionModelRuntime = {
|
|
653
|
+
...modelRuntime,
|
|
654
|
+
executionId: runner.id
|
|
639
655
|
};
|
|
640
|
-
|
|
641
|
-
const
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
}
|
|
654
|
-
], modelRuntime, modelRuntime, {
|
|
656
|
+
const { timeoutMs: _timeoutMs, checkIntervalMs: _checkIntervalMs, abortSignal: _abortSignal, domIncluded, screenshotIncluded, ...restOpt } = opt;
|
|
657
|
+
const serviceExtractOpt = {
|
|
658
|
+
domIncluded,
|
|
659
|
+
screenshotIncluded,
|
|
660
|
+
...restOpt
|
|
661
|
+
};
|
|
662
|
+
assert(assertion, 'No assertion for waitFor');
|
|
663
|
+
const overallStartTime = Date.now();
|
|
664
|
+
let errorThought = '';
|
|
665
|
+
while(Date.now() - overallStartTime < timeoutMs){
|
|
666
|
+
const currentCheckStart = Date.now();
|
|
667
|
+
abortSignal?.throwIfAborted();
|
|
668
|
+
const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
|
|
655
669
|
abortSignal
|
|
656
670
|
});
|
|
657
|
-
|
|
671
|
+
const result = await session.appendAndRun(queryTask);
|
|
672
|
+
abortSignal.throwIfAborted();
|
|
673
|
+
if (Date.now() >= deadlineAt) {
|
|
674
|
+
deadline.abort(new Error(`waitFor timeout after ${timeoutMs}ms`));
|
|
675
|
+
abortSignal.throwIfAborted();
|
|
676
|
+
}
|
|
677
|
+
if (result?.output) return {
|
|
678
|
+
output: void 0,
|
|
679
|
+
runner
|
|
680
|
+
};
|
|
681
|
+
errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
|
|
682
|
+
const now = Date.now();
|
|
683
|
+
if (now - currentCheckStart < checkIntervalMs) {
|
|
684
|
+
const elapsed = now - currentCheckStart;
|
|
685
|
+
const timeRemaining = Math.max(0, Math.min(checkIntervalMs - elapsed, timeoutMs - (Date.now() - overallStartTime)));
|
|
686
|
+
const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
|
|
687
|
+
const { tasks: sleepTasks } = await this.convertPlanToExecutable([
|
|
688
|
+
{
|
|
689
|
+
type: 'Sleep',
|
|
690
|
+
param: {
|
|
691
|
+
timeMs: timeRemaining
|
|
692
|
+
},
|
|
693
|
+
thought
|
|
694
|
+
}
|
|
695
|
+
], modelRuntime, modelRuntime, {
|
|
696
|
+
abortSignal
|
|
697
|
+
});
|
|
698
|
+
if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
|
|
699
|
+
}
|
|
658
700
|
}
|
|
701
|
+
return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
|
|
702
|
+
} finally{
|
|
703
|
+
clearTimeout(timer);
|
|
659
704
|
}
|
|
660
|
-
return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
|
|
661
705
|
}
|
|
662
706
|
constructor(interfaceInstance, service, opts){
|
|
663
707
|
_define_property(this, "interface", void 0);
|
|
@@ -665,6 +709,7 @@ class TaskExecutor {
|
|
|
665
709
|
_define_property(this, "taskCache", void 0);
|
|
666
710
|
_define_property(this, "getDefaultModelRuntime", void 0);
|
|
667
711
|
_define_property(this, "providedActionSpace", void 0);
|
|
712
|
+
_define_property(this, "planningWaitFor", void 0);
|
|
668
713
|
_define_property(this, "taskBuilder", void 0);
|
|
669
714
|
_define_property(this, "onTaskStartCallback", void 0);
|
|
670
715
|
_define_property(this, "hooks", void 0);
|
|
@@ -678,6 +723,7 @@ class TaskExecutor {
|
|
|
678
723
|
this.taskCache = opts.taskCache;
|
|
679
724
|
this.onTaskStartCallback = opts?.onTaskStart;
|
|
680
725
|
this.replanningCycleLimit = opts.replanningCycleLimit;
|
|
726
|
+
this.planningWaitFor = opts.planningWaitFor ?? false;
|
|
681
727
|
this.waitAfterAction = opts.waitAfterAction;
|
|
682
728
|
this.useDeviceTime = opts.useDeviceTime;
|
|
683
729
|
this.hooks = opts.hooks;
|