@rpascene/core 0.30.18 → 0.30.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +6 -6
- package/dist/es/agent/task-cache.mjs +0 -2
- package/dist/es/agent/tasks.mjs +14 -49
- package/dist/es/agent/ui-utils.mjs +0 -2
- package/dist/es/agent/utils.mjs +1 -3
- package/dist/es/ai-model/action-executor.mjs +6 -7
- package/dist/es/ai-model/common.mjs +0 -2
- package/dist/es/ai-model/conversation-history.mjs +0 -2
- package/dist/es/ai-model/inspect.mjs +6 -8
- package/dist/es/ai-model/llm-planning.mjs +2 -4
- package/dist/es/ai-model/prompt/assertion.mjs +0 -2
- package/dist/es/ai-model/prompt/common.mjs +0 -2
- package/dist/es/ai-model/prompt/describe.mjs +0 -2
- package/dist/es/ai-model/prompt/extraction.mjs +0 -2
- package/dist/es/ai-model/prompt/llm-locator.mjs +0 -2
- package/dist/es/ai-model/prompt/llm-planning.mjs +0 -2
- package/dist/es/ai-model/prompt/llm-section-locator.mjs +0 -2
- package/dist/es/ai-model/prompt/playwright-generator.mjs +6 -7
- package/dist/es/ai-model/prompt/ui-tars-locator.mjs +0 -2
- package/dist/es/ai-model/prompt/ui-tars-planning.mjs +0 -2
- package/dist/es/ai-model/prompt/util.mjs +0 -2
- package/dist/es/ai-model/prompt/yaml-generator.mjs +6 -7
- package/dist/es/ai-model/service-caller/index.mjs +11 -8
- package/dist/es/ai-model/ui-tars-planning.mjs +2 -4
- package/dist/es/device/index.mjs +0 -2
- package/dist/es/index.mjs +0 -2
- package/dist/es/insight/index.mjs +9 -8
- package/dist/es/insight/utils.mjs +0 -2
- package/dist/es/report.mjs +0 -2
- package/dist/es/types.mjs +0 -2
- package/dist/es/utils.mjs +2 -4
- package/dist/es/yaml/builder.mjs +0 -2
- package/dist/es/yaml/player.mjs +0 -2
- package/dist/es/yaml/utils.mjs +0 -2
- package/dist/lib/agent/agent.js +6 -6
- package/dist/lib/agent/index.js +0 -2
- package/dist/lib/agent/task-cache.js +0 -2
- package/dist/lib/agent/tasks.js +14 -49
- package/dist/lib/agent/ui-utils.js +0 -2
- package/dist/lib/agent/utils.js +1 -3
- package/dist/lib/ai-model/action-executor.js +6 -7
- package/dist/lib/ai-model/common.js +0 -2
- package/dist/lib/ai-model/conversation-history.js +0 -2
- package/dist/lib/ai-model/index.js +0 -2
- package/dist/lib/ai-model/inspect.js +6 -8
- package/dist/lib/ai-model/llm-planning.js +2 -4
- package/dist/lib/ai-model/prompt/assertion.js +0 -2
- package/dist/lib/ai-model/prompt/common.js +0 -2
- package/dist/lib/ai-model/prompt/describe.js +0 -2
- package/dist/lib/ai-model/prompt/extraction.js +0 -2
- package/dist/lib/ai-model/prompt/llm-locator.js +0 -2
- package/dist/lib/ai-model/prompt/llm-planning.js +0 -2
- package/dist/lib/ai-model/prompt/llm-section-locator.js +0 -2
- package/dist/lib/ai-model/prompt/playwright-generator.js +6 -7
- package/dist/lib/ai-model/prompt/ui-tars-locator.js +0 -2
- package/dist/lib/ai-model/prompt/ui-tars-planning.js +0 -2
- package/dist/lib/ai-model/prompt/util.js +0 -2
- package/dist/lib/ai-model/prompt/yaml-generator.js +6 -7
- package/dist/lib/ai-model/service-caller/index.js +11 -8
- package/dist/lib/ai-model/ui-tars-planning.js +2 -4
- package/dist/lib/device/index.js +0 -2
- package/dist/lib/image/index.js +0 -2
- package/dist/lib/index.js +0 -2
- package/dist/lib/insight/index.js +9 -8
- package/dist/lib/insight/utils.js +0 -2
- package/dist/lib/report.js +0 -2
- package/dist/lib/tree.js +0 -2
- package/dist/lib/types.js +0 -2
- package/dist/lib/utils.js +2 -4
- package/dist/lib/yaml/builder.js +0 -2
- package/dist/lib/yaml/index.js +0 -2
- package/dist/lib/yaml/player.js +0 -2
- package/dist/lib/yaml/utils.js +0 -2
- package/dist/lib/yaml.js +0 -2
- package/dist/types/agent/agent.d.ts +2 -0
- package/dist/types/agent/tasks.d.ts +2 -6
- package/dist/types/ai-model/action-executor.d.ts +2 -2
- package/dist/types/ai-model/common.d.ts +1 -1
- package/dist/types/ai-model/inspect.d.ts +3 -0
- package/dist/types/ai-model/llm-planning.d.ts +1 -0
- package/dist/types/ai-model/prompt/playwright-generator.d.ts +2 -2
- package/dist/types/ai-model/prompt/yaml-generator.d.ts +2 -2
- package/dist/types/ai-model/service-caller/index.d.ts +3 -2
- package/dist/types/ai-model/ui-tars-planning.d.ts +1 -0
- package/dist/types/insight/index.d.ts +3 -2
- package/dist/types/types.d.ts +1 -0
- package/package.json +3 -3
- package/dist/es/agent/agent.mjs.map +0 -1
- package/dist/es/agent/task-cache.mjs.map +0 -1
- package/dist/es/agent/tasks.mjs.map +0 -1
- package/dist/es/agent/ui-utils.mjs.map +0 -1
- package/dist/es/agent/utils.mjs.map +0 -1
- package/dist/es/ai-model/action-executor.mjs.map +0 -1
- package/dist/es/ai-model/common.mjs.map +0 -1
- package/dist/es/ai-model/conversation-history.mjs.map +0 -1
- package/dist/es/ai-model/inspect.mjs.map +0 -1
- package/dist/es/ai-model/llm-planning.mjs.map +0 -1
- package/dist/es/ai-model/prompt/assertion.mjs.map +0 -1
- package/dist/es/ai-model/prompt/common.mjs.map +0 -1
- package/dist/es/ai-model/prompt/describe.mjs.map +0 -1
- package/dist/es/ai-model/prompt/extraction.mjs.map +0 -1
- package/dist/es/ai-model/prompt/llm-locator.mjs.map +0 -1
- package/dist/es/ai-model/prompt/llm-planning.mjs.map +0 -1
- package/dist/es/ai-model/prompt/llm-section-locator.mjs.map +0 -1
- package/dist/es/ai-model/prompt/playwright-generator.mjs.map +0 -1
- package/dist/es/ai-model/prompt/ui-tars-locator.mjs.map +0 -1
- package/dist/es/ai-model/prompt/ui-tars-planning.mjs.map +0 -1
- package/dist/es/ai-model/prompt/util.mjs.map +0 -1
- package/dist/es/ai-model/prompt/yaml-generator.mjs.map +0 -1
- package/dist/es/ai-model/service-caller/index.mjs.map +0 -1
- package/dist/es/ai-model/ui-tars-planning.mjs.map +0 -1
- package/dist/es/device/index.mjs.map +0 -1
- package/dist/es/index.mjs.map +0 -1
- package/dist/es/insight/index.mjs.map +0 -1
- package/dist/es/insight/utils.mjs.map +0 -1
- package/dist/es/report.mjs.map +0 -1
- package/dist/es/types.mjs.map +0 -1
- package/dist/es/utils.mjs.map +0 -1
- package/dist/es/yaml/builder.mjs.map +0 -1
- package/dist/es/yaml/player.mjs.map +0 -1
- package/dist/es/yaml/utils.mjs.map +0 -1
- package/dist/lib/agent/agent.js.map +0 -1
- package/dist/lib/agent/index.js.map +0 -1
- package/dist/lib/agent/task-cache.js.map +0 -1
- package/dist/lib/agent/tasks.js.map +0 -1
- package/dist/lib/agent/ui-utils.js.map +0 -1
- package/dist/lib/agent/utils.js.map +0 -1
- package/dist/lib/ai-model/action-executor.js.map +0 -1
- package/dist/lib/ai-model/common.js.map +0 -1
- package/dist/lib/ai-model/conversation-history.js.map +0 -1
- package/dist/lib/ai-model/index.js.map +0 -1
- package/dist/lib/ai-model/inspect.js.map +0 -1
- package/dist/lib/ai-model/llm-planning.js.map +0 -1
- package/dist/lib/ai-model/prompt/assertion.js.map +0 -1
- package/dist/lib/ai-model/prompt/common.js.map +0 -1
- package/dist/lib/ai-model/prompt/describe.js.map +0 -1
- package/dist/lib/ai-model/prompt/extraction.js.map +0 -1
- package/dist/lib/ai-model/prompt/llm-locator.js.map +0 -1
- package/dist/lib/ai-model/prompt/llm-planning.js.map +0 -1
- package/dist/lib/ai-model/prompt/llm-section-locator.js.map +0 -1
- package/dist/lib/ai-model/prompt/playwright-generator.js.map +0 -1
- package/dist/lib/ai-model/prompt/ui-tars-locator.js.map +0 -1
- package/dist/lib/ai-model/prompt/ui-tars-planning.js.map +0 -1
- package/dist/lib/ai-model/prompt/util.js.map +0 -1
- package/dist/lib/ai-model/prompt/yaml-generator.js.map +0 -1
- package/dist/lib/ai-model/service-caller/index.js.map +0 -1
- package/dist/lib/ai-model/ui-tars-planning.js.map +0 -1
- package/dist/lib/device/index.js.map +0 -1
- package/dist/lib/image/index.js.map +0 -1
- package/dist/lib/index.js.map +0 -1
- package/dist/lib/insight/index.js.map +0 -1
- package/dist/lib/insight/utils.js.map +0 -1
- package/dist/lib/report.js.map +0 -1
- package/dist/lib/tree.js.map +0 -1
- package/dist/lib/types.js.map +0 -1
- package/dist/lib/utils.js.map +0 -1
- package/dist/lib/yaml/builder.js.map +0 -1
- package/dist/lib/yaml/index.js.map +0 -1
- package/dist/lib/yaml/player.js.map +0 -1
- package/dist/lib/yaml/utils.js.map +0 -1
- package/dist/lib/yaml.js.map +0 -1
package/dist/es/yaml/builder.mjs
CHANGED
package/dist/es/yaml/player.mjs
CHANGED
package/dist/es/yaml/utils.mjs
CHANGED
package/dist/lib/agent/agent.js
CHANGED
|
@@ -324,6 +324,8 @@ class Agent {
|
|
|
324
324
|
}
|
|
325
325
|
async aiAction(taskPrompt, opt) {
|
|
326
326
|
var _this_taskCache, _this_taskCache1;
|
|
327
|
+
const abortSignal = null == opt ? void 0 : opt.abortSignal;
|
|
328
|
+
if (null == abortSignal ? void 0 : abortSignal.aborted) throw new Error(`aiAct aborted: ${abortSignal.reason || 'signal already aborted'}`);
|
|
327
329
|
const modelConfig = this.modelConfigManager.getModelConfig('planning');
|
|
328
330
|
const cacheable = null == opt ? void 0 : opt.cacheable;
|
|
329
331
|
const isVlmUiTars = 'vlm-ui-tars' === modelConfig.vlMode;
|
|
@@ -336,7 +338,7 @@ class Agent {
|
|
|
336
338
|
const yaml = null == (_matchedCache_cacheContent1 = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent1.yamlWorkflow;
|
|
337
339
|
return this.runYaml(yaml);
|
|
338
340
|
}
|
|
339
|
-
const { output, executor } = await this.taskExecutor.action(taskPrompt, modelConfig, this.opts.aiActionContext, cacheable);
|
|
341
|
+
const { output, executor } = await this.taskExecutor.action(taskPrompt, modelConfig, this.opts.aiActionContext, cacheable, abortSignal);
|
|
340
342
|
console.log(output, 'output');
|
|
341
343
|
if (this.taskCache && (null == output ? void 0 : output.yamlFlow) && false !== cacheable) {
|
|
342
344
|
const yamlContent = {
|
|
@@ -388,7 +390,7 @@ class Agent {
|
|
|
388
390
|
return this.aiString(prompt, opt);
|
|
389
391
|
}
|
|
390
392
|
async describeElementAtPoint(center, opt) {
|
|
391
|
-
const { verifyPrompt = true, retryLimit = 3 } = opt || {};
|
|
393
|
+
const { verifyPrompt = true, retryLimit = 3, abortSignal } = opt || {};
|
|
392
394
|
let success = false;
|
|
393
395
|
let retryCount = 0;
|
|
394
396
|
let resultPrompt = '';
|
|
@@ -399,7 +401,8 @@ class Agent {
|
|
|
399
401
|
debug('aiDescribe', center, 'verifyPrompt', verifyPrompt, 'retryCount', retryCount, 'deepThink', deepThink);
|
|
400
402
|
const modelConfig = this.modelConfigManager.getModelConfig('grounding');
|
|
401
403
|
const text = await this.insight.describe(center, modelConfig, {
|
|
402
|
-
deepThink
|
|
404
|
+
deepThink,
|
|
405
|
+
abortSignal
|
|
403
406
|
});
|
|
404
407
|
debug('aiDescribe text', text);
|
|
405
408
|
(0, utils_namespaceObject.assert)(text.description, `failed to describe element at [${center}]`);
|
|
@@ -517,7 +520,6 @@ class Agent {
|
|
|
517
520
|
async destroy() {
|
|
518
521
|
var _this_interface_destroy, _this_interface;
|
|
519
522
|
if (this.destroyed) return;
|
|
520
|
-
this.taskExecutor.cancelPlanning();
|
|
521
523
|
await (null == (_this_interface_destroy = (_this_interface = this.interface).destroy) ? void 0 : _this_interface_destroy.call(_this_interface));
|
|
522
524
|
this.resetDump();
|
|
523
525
|
this.destroyed = true;
|
|
@@ -680,5 +682,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
680
682
|
Object.defineProperty(exports, '__esModule', {
|
|
681
683
|
value: true
|
|
682
684
|
});
|
|
683
|
-
|
|
684
|
-
//# sourceMappingURL=agent.js.map
|
package/dist/lib/agent/index.js
CHANGED
package/dist/lib/agent/tasks.js
CHANGED
|
@@ -110,7 +110,7 @@ class TaskExecutor {
|
|
|
110
110
|
thought: plan.thought,
|
|
111
111
|
executor: async (param, taskContext)=>{
|
|
112
112
|
var _this_taskCache, _locateCacheRecord_cacheContent;
|
|
113
|
-
const { task } = taskContext;
|
|
113
|
+
const { task, abortSignal } = taskContext;
|
|
114
114
|
(0, utils_namespaceObject.assert)((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
|
|
115
115
|
let insightDump;
|
|
116
116
|
let usage;
|
|
@@ -148,7 +148,7 @@ class TaskExecutor {
|
|
|
148
148
|
const planHitFlag = !!elementFromPlan;
|
|
149
149
|
const elementFromAiLocate = userExpectedPathHitFlag || cacheHitFlag || planHitFlag ? void 0 : (await this.insight.locate(param, {
|
|
150
150
|
context: uiContext
|
|
151
|
-
}, modelConfig)).element;
|
|
151
|
+
}, modelConfig, abortSignal)).element;
|
|
152
152
|
const aiLocateHitFlag = !!elementFromAiLocate;
|
|
153
153
|
const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
|
|
154
154
|
if (this.interface.getElementFromPoint && element.center) try {
|
|
@@ -494,44 +494,21 @@ class TaskExecutor {
|
|
|
494
494
|
getReplanningCycleLimit(isVlmUiTars) {
|
|
495
495
|
return this.replanningCycleLimit || env_namespaceObject.globalConfigManager.getEnvConfigInNumber(env_namespaceObject.MIDSCENE_REPLANNING_CYCLE_LIMIT) || (isVlmUiTars ? defaultVlmUiTarsReplanningCycleLimit : defaultReplanningCycleLimit);
|
|
496
496
|
}
|
|
497
|
-
|
|
498
|
-
var _this_abortController;
|
|
499
|
-
console.log("\u53D6\u6D88\u6267\u884C\u89C4\u5212");
|
|
500
|
-
this.isPlanning = false;
|
|
501
|
-
null == (_this_abortController = this.abortController) || _this_abortController.abort();
|
|
502
|
-
}
|
|
503
|
-
cleanupPlanning() {
|
|
504
|
-
var _this_abortController;
|
|
505
|
-
null == (_this_abortController = this.abortController) || _this_abortController.abort();
|
|
506
|
-
this.isPlanning = false;
|
|
507
|
-
}
|
|
508
|
-
isActive() {
|
|
509
|
-
return this.isPlanning;
|
|
510
|
-
}
|
|
511
|
-
async action(userPrompt, modelConfig, actionContext, cacheable) {
|
|
512
|
-
console.log(actionContext, 'actionContext--------------');
|
|
513
|
-
if (this.isPlanning) {
|
|
514
|
-
var _this_abortController;
|
|
515
|
-
console.warn('Executor is already running');
|
|
516
|
-
null == (_this_abortController = this.abortController) || _this_abortController.abort();
|
|
517
|
-
}
|
|
497
|
+
async action(userPrompt, modelConfig, actionContext, cacheable, abortSignal) {
|
|
518
498
|
this.conversationHistory.reset();
|
|
519
499
|
const userPromptList = userPrompt.split('\n').map((item)=>item.trim()).filter(Boolean);
|
|
520
500
|
let nextPrompt = userPromptList.shift();
|
|
521
|
-
const abortController = new AbortController();
|
|
522
|
-
this.abortController = abortController;
|
|
523
501
|
const taskExecutor = new action_executor_js_namespaceObject.Executor((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Action', userPrompt), {
|
|
524
502
|
onTaskStart: this.onTaskStartCallback,
|
|
525
|
-
|
|
503
|
+
abortSignal
|
|
526
504
|
});
|
|
527
|
-
this.isPlanning = true;
|
|
528
505
|
let replanCount = 0;
|
|
529
506
|
const yamlFlow = [];
|
|
530
507
|
const replanningCycleLimit = this.getReplanningCycleLimit('vlm-ui-tars' === modelConfig.vlMode);
|
|
531
|
-
while(
|
|
508
|
+
while(nextPrompt){
|
|
532
509
|
var _result_output;
|
|
533
510
|
console.log('action while running', nextPrompt);
|
|
534
|
-
if (
|
|
511
|
+
if (null == abortSignal ? void 0 : abortSignal.aborted) return {
|
|
535
512
|
output: void 0,
|
|
536
513
|
executor: taskExecutor
|
|
537
514
|
};
|
|
@@ -544,14 +521,10 @@ class TaskExecutor {
|
|
|
544
521
|
const result = await taskExecutor.flush();
|
|
545
522
|
console.dir(null == result ? void 0 : null == (_result_output = result.output) ? void 0 : _result_output.actions, 'result?.output?.actions');
|
|
546
523
|
const planResult = null == result ? void 0 : result.output;
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
output: planResult,
|
|
552
|
-
executor: taskExecutor
|
|
553
|
-
};
|
|
554
|
-
}
|
|
524
|
+
if (taskExecutor.isInErrorState()) return {
|
|
525
|
+
output: planResult,
|
|
526
|
+
executor: taskExecutor
|
|
527
|
+
};
|
|
555
528
|
const plans = (null == planResult ? void 0 : planResult.actions) || [];
|
|
556
529
|
yamlFlow.push(...(null == planResult ? void 0 : planResult.yamlFlow) || []);
|
|
557
530
|
let executables;
|
|
@@ -559,17 +532,13 @@ class TaskExecutor {
|
|
|
559
532
|
executables = await this.convertPlanToExecutable(plans, modelConfig, cacheable);
|
|
560
533
|
taskExecutor.append(executables.tasks);
|
|
561
534
|
} catch (error) {
|
|
562
|
-
this.cleanupPlanning();
|
|
563
535
|
return this.appendErrorPlan(taskExecutor, `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`, modelConfig);
|
|
564
536
|
}
|
|
565
537
|
await taskExecutor.flush();
|
|
566
|
-
if (taskExecutor.isInErrorState()) {
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
executor: taskExecutor
|
|
571
|
-
};
|
|
572
|
-
}
|
|
538
|
+
if (taskExecutor.isInErrorState()) return {
|
|
539
|
+
output: void 0,
|
|
540
|
+
executor: taskExecutor
|
|
541
|
+
};
|
|
573
542
|
if (null == planResult ? void 0 : planResult.more_actions_needed_by_instruction) replanCount++;
|
|
574
543
|
else {
|
|
575
544
|
this.conversationHistory.reset();
|
|
@@ -577,7 +546,6 @@ class TaskExecutor {
|
|
|
577
546
|
replanCount = 0;
|
|
578
547
|
}
|
|
579
548
|
}
|
|
580
|
-
this.cleanupPlanning();
|
|
581
549
|
return {
|
|
582
550
|
output: {
|
|
583
551
|
yamlFlow
|
|
@@ -740,7 +708,6 @@ class TaskExecutor {
|
|
|
740
708
|
_define_property(this, "onTaskStartCallback", void 0);
|
|
741
709
|
_define_property(this, "replanningCycleLimit", void 0);
|
|
742
710
|
_define_property(this, "isPlanning", false);
|
|
743
|
-
_define_property(this, "abortController", null);
|
|
744
711
|
this.interface = interfaceInstance;
|
|
745
712
|
this.insight = insight;
|
|
746
713
|
this.taskCache = opts.taskCache;
|
|
@@ -758,5 +725,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
758
725
|
Object.defineProperty(exports, '__esModule', {
|
|
759
726
|
value: true
|
|
760
727
|
});
|
|
761
|
-
|
|
762
|
-
//# sourceMappingURL=tasks.js.map
|
package/dist/lib/agent/utils.js
CHANGED
|
@@ -190,7 +190,7 @@ function trimContextByViewport(execution) {
|
|
|
190
190
|
}) : execution.tasks
|
|
191
191
|
};
|
|
192
192
|
}
|
|
193
|
-
const getRpasceneVersion = ()=>"0.30.
|
|
193
|
+
const getRpasceneVersion = ()=>"0.30.19";
|
|
194
194
|
const parsePrompt = (prompt)=>{
|
|
195
195
|
if ('string' == typeof prompt) return {
|
|
196
196
|
textPrompt: prompt,
|
|
@@ -229,5 +229,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
229
229
|
Object.defineProperty(exports, '__esModule', {
|
|
230
230
|
value: true
|
|
231
231
|
});
|
|
232
|
-
|
|
233
|
-
//# sourceMappingURL=utils.js.map
|
|
@@ -70,8 +70,8 @@ class Executor {
|
|
|
70
70
|
start: Date.now()
|
|
71
71
|
};
|
|
72
72
|
try {
|
|
73
|
-
var
|
|
74
|
-
if (null == (
|
|
73
|
+
var _this_abortSignal;
|
|
74
|
+
if (null == (_this_abortSignal = this.abortSignal) ? void 0 : _this_abortSignal.aborted) throw 'Operation stopped';
|
|
75
75
|
task.status = 'running';
|
|
76
76
|
try {
|
|
77
77
|
if (this.onTaskStart) await this.onTaskStart(task);
|
|
@@ -88,7 +88,8 @@ class Executor {
|
|
|
88
88
|
let returnValue;
|
|
89
89
|
const executorContext = {
|
|
90
90
|
task,
|
|
91
|
-
element: null == previousFindOutput ? void 0 : previousFindOutput.element
|
|
91
|
+
element: null == previousFindOutput ? void 0 : previousFindOutput.element,
|
|
92
|
+
abortSignal: this.abortSignal
|
|
92
93
|
};
|
|
93
94
|
if ('Insight' === task.type) {
|
|
94
95
|
(0, utils_namespaceObject.assert)('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported insight subType: ${task.subType}`);
|
|
@@ -149,12 +150,12 @@ class Executor {
|
|
|
149
150
|
_define_property(this, "tasks", void 0);
|
|
150
151
|
_define_property(this, "status", void 0);
|
|
151
152
|
_define_property(this, "onTaskStart", void 0);
|
|
152
|
-
_define_property(this, "
|
|
153
|
+
_define_property(this, "abortSignal", void 0);
|
|
153
154
|
this.status = (null == options ? void 0 : options.tasks) && options.tasks.length > 0 ? 'pending' : 'init';
|
|
154
155
|
this.name = name;
|
|
155
156
|
this.tasks = ((null == options ? void 0 : options.tasks) || []).map((item)=>this.markTaskAsPending(item));
|
|
156
157
|
this.onTaskStart = null == options ? void 0 : options.onTaskStart;
|
|
157
|
-
this.
|
|
158
|
+
this.abortSignal = null == options ? void 0 : options.abortSignal;
|
|
158
159
|
}
|
|
159
160
|
}
|
|
160
161
|
exports.Executor = __webpack_exports__.Executor;
|
|
@@ -164,5 +165,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
164
165
|
Object.defineProperty(exports, '__esModule', {
|
|
165
166
|
value: true
|
|
166
167
|
});
|
|
167
|
-
|
|
168
|
-
//# sourceMappingURL=action-executor.js.map
|
|
@@ -84,7 +84,7 @@ const promptsToChatParam = async (multimodalPrompt)=>{
|
|
|
84
84
|
return msgs;
|
|
85
85
|
};
|
|
86
86
|
async function AiLocateElement(options) {
|
|
87
|
-
const { context, targetElementDescription, callAIFn, modelConfig } = options;
|
|
87
|
+
const { context, targetElementDescription, callAIFn, modelConfig, abortSignal } = options;
|
|
88
88
|
const { vlMode } = modelConfig;
|
|
89
89
|
const { screenshotBase64 } = context;
|
|
90
90
|
const { description, elementById, insertElementByPosition } = await (0, util_js_namespaceObject.describeUserPage)(context, {
|
|
@@ -146,7 +146,7 @@ async function AiLocateElement(options) {
|
|
|
146
146
|
});
|
|
147
147
|
msgs.push(...addOns);
|
|
148
148
|
}
|
|
149
|
-
const res = await callAIFn(msgs, external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
|
|
149
|
+
const res = await callAIFn(msgs, external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig, abortSignal);
|
|
150
150
|
const rawResponse = JSON.stringify(res.content);
|
|
151
151
|
let resRect;
|
|
152
152
|
let matchedElements = 'elements' in res.content ? res.content.elements : [];
|
|
@@ -193,7 +193,7 @@ async function AiLocateElement(options) {
|
|
|
193
193
|
};
|
|
194
194
|
}
|
|
195
195
|
async function AiLocateSection(options) {
|
|
196
|
-
const { context, sectionDescription, modelConfig } = options;
|
|
196
|
+
const { context, sectionDescription, modelConfig, abortSignal } = options;
|
|
197
197
|
const { vlMode } = modelConfig;
|
|
198
198
|
const { screenshotBase64 } = context;
|
|
199
199
|
const systemPrompt = (0, llm_section_locator_js_namespaceObject.systemPromptToLocateSection)(vlMode);
|
|
@@ -229,7 +229,7 @@ async function AiLocateSection(options) {
|
|
|
229
229
|
});
|
|
230
230
|
msgs.push(...addOns);
|
|
231
231
|
}
|
|
232
|
-
const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
|
|
232
|
+
const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig, abortSignal);
|
|
233
233
|
let sectionRect;
|
|
234
234
|
const sectionBbox = result.content.bbox;
|
|
235
235
|
if (sectionBbox) {
|
|
@@ -263,7 +263,7 @@ async function AiLocateSection(options) {
|
|
|
263
263
|
};
|
|
264
264
|
}
|
|
265
265
|
async function AiExtractElementInfo(options) {
|
|
266
|
-
const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
|
|
266
|
+
const { dataQuery, context, extractOption, multimodalPrompt, modelConfig, abortSignal } = options;
|
|
267
267
|
const { vlMode } = modelConfig;
|
|
268
268
|
const systemPrompt = (0, extraction_js_namespaceObject.systemPromptToExtract)();
|
|
269
269
|
const { screenshotBase64 } = context;
|
|
@@ -304,7 +304,7 @@ async function AiExtractElementInfo(options) {
|
|
|
304
304
|
});
|
|
305
305
|
msgs.push(...addOns);
|
|
306
306
|
}
|
|
307
|
-
const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
|
|
307
|
+
const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig, abortSignal);
|
|
308
308
|
return {
|
|
309
309
|
parseResult: result.content,
|
|
310
310
|
elementById,
|
|
@@ -322,5 +322,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
322
322
|
Object.defineProperty(exports, '__esModule', {
|
|
323
323
|
value: true
|
|
324
324
|
});
|
|
325
|
-
|
|
326
|
-
//# sourceMappingURL=inspect.js.map
|
|
@@ -36,7 +36,7 @@ const index_js_namespaceObject = require("./service-caller/index.js");
|
|
|
36
36
|
const debug = (0, logger_namespaceObject.getDebug)('planning');
|
|
37
37
|
async function plan(userInstruction, opts) {
|
|
38
38
|
var _opts_conversationHistory, _planFromAI_action;
|
|
39
|
-
const { context, modelConfig, conversationHistory } = opts;
|
|
39
|
+
const { context, modelConfig, conversationHistory, abortSignal } = opts;
|
|
40
40
|
const { screenshotBase64, size } = context;
|
|
41
41
|
const { modelName, vlMode } = modelConfig;
|
|
42
42
|
const { description: pageDescription, elementById } = await (0, util_js_namespaceObject.describeUserPage)(context, {
|
|
@@ -112,7 +112,7 @@ async function plan(userInstruction, opts) {
|
|
|
112
112
|
]
|
|
113
113
|
}
|
|
114
114
|
];
|
|
115
|
-
const { content, usage } = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.PLAN, modelConfig);
|
|
115
|
+
const { content, usage } = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.PLAN, modelConfig, abortSignal);
|
|
116
116
|
const rawResponse = JSON.stringify(content, void 0, 2);
|
|
117
117
|
const planFromAI = content;
|
|
118
118
|
const actions = ((null == (_planFromAI_action = planFromAI.action) ? void 0 : _planFromAI_action.type) ? [
|
|
@@ -170,5 +170,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
170
170
|
Object.defineProperty(exports, '__esModule', {
|
|
171
171
|
value: true
|
|
172
172
|
});
|
|
173
|
-
|
|
174
|
-
//# sourceMappingURL=llm-planning.js.map
|
|
@@ -38,7 +38,7 @@ __webpack_require__.d(__webpack_exports__, {
|
|
|
38
38
|
const constants_namespaceObject = require("@rpascene/shared/constants");
|
|
39
39
|
const external_index_js_namespaceObject = require("../index.js");
|
|
40
40
|
const external_yaml_generator_js_namespaceObject = require("./yaml-generator.js");
|
|
41
|
-
const generatePlaywrightTest = async (events, options, modelConfig)=>{
|
|
41
|
+
const generatePlaywrightTest = async (events, options, modelConfig, abortSignal)=>{
|
|
42
42
|
(0, external_yaml_generator_js_namespaceObject.validateEvents)(events);
|
|
43
43
|
const summary = (0, external_yaml_generator_js_namespaceObject.prepareEventSummary)(events, {
|
|
44
44
|
testName: options.testName,
|
|
@@ -84,11 +84,11 @@ ${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
|
|
|
84
84
|
content: messageContent
|
|
85
85
|
}
|
|
86
86
|
];
|
|
87
|
-
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig);
|
|
87
|
+
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, abortSignal);
|
|
88
88
|
if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return response.content;
|
|
89
89
|
throw new Error('Failed to generate Playwright test code');
|
|
90
90
|
};
|
|
91
|
-
const generatePlaywrightTestStream = async (events, options, modelConfig)=>{
|
|
91
|
+
const generatePlaywrightTestStream = async (events, options, modelConfig, abortSignal)=>{
|
|
92
92
|
(0, external_yaml_generator_js_namespaceObject.validateEvents)(events);
|
|
93
93
|
const summary = (0, external_yaml_generator_js_namespaceObject.prepareEventSummary)(events, {
|
|
94
94
|
testName: options.testName,
|
|
@@ -137,10 +137,11 @@ ${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
|
|
|
137
137
|
];
|
|
138
138
|
if (options.stream && options.onChunk) return await (0, external_index_js_namespaceObject.callAI)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, {
|
|
139
139
|
stream: true,
|
|
140
|
-
onChunk: options.onChunk
|
|
140
|
+
onChunk: options.onChunk,
|
|
141
|
+
abortSignal
|
|
141
142
|
});
|
|
142
143
|
{
|
|
143
|
-
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig);
|
|
144
|
+
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, abortSignal);
|
|
144
145
|
if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return {
|
|
145
146
|
content: response.content,
|
|
146
147
|
usage: response.usage,
|
|
@@ -174,5 +175,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
174
175
|
Object.defineProperty(exports, '__esModule', {
|
|
175
176
|
value: true
|
|
176
177
|
});
|
|
177
|
-
|
|
178
|
-
//# sourceMappingURL=playwright-generator.js.map
|
|
@@ -129,7 +129,7 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
|
|
|
129
129
|
const validateEvents = (events)=>{
|
|
130
130
|
if (!events.length) throw new Error('No events provided for test generation');
|
|
131
131
|
};
|
|
132
|
-
const generateYamlTest = async (events, options, modelConfig)=>{
|
|
132
|
+
const generateYamlTest = async (events, options, modelConfig, abortSignal)=>{
|
|
133
133
|
try {
|
|
134
134
|
validateEvents(events);
|
|
135
135
|
const summary = prepareEventSummary(events, {
|
|
@@ -178,14 +178,14 @@ Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdow
|
|
|
178
178
|
}))
|
|
179
179
|
});
|
|
180
180
|
}
|
|
181
|
-
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig);
|
|
181
|
+
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, abortSignal);
|
|
182
182
|
if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return response.content;
|
|
183
183
|
throw new Error('Failed to generate YAML test configuration');
|
|
184
184
|
} catch (error) {
|
|
185
185
|
throw new Error(`Failed to generate YAML test: ${error}`);
|
|
186
186
|
}
|
|
187
187
|
};
|
|
188
|
-
const generateYamlTestStream = async (events, options, modelConfig)=>{
|
|
188
|
+
const generateYamlTestStream = async (events, options, modelConfig, abortSignal)=>{
|
|
189
189
|
try {
|
|
190
190
|
validateEvents(events);
|
|
191
191
|
const summary = prepareEventSummary(events, {
|
|
@@ -236,10 +236,11 @@ Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdow
|
|
|
236
236
|
}
|
|
237
237
|
if (options.stream && options.onChunk) return await (0, external_index_js_namespaceObject.callAI)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, {
|
|
238
238
|
stream: true,
|
|
239
|
-
onChunk: options.onChunk
|
|
239
|
+
onChunk: options.onChunk,
|
|
240
|
+
abortSignal
|
|
240
241
|
});
|
|
241
242
|
{
|
|
242
|
-
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig);
|
|
243
|
+
const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.TEXT, modelConfig, abortSignal);
|
|
243
244
|
if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return {
|
|
244
245
|
content: response.content,
|
|
245
246
|
usage: response.usage,
|
|
@@ -276,5 +277,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
276
277
|
Object.defineProperty(exports, '__esModule', {
|
|
277
278
|
value: true
|
|
278
279
|
});
|
|
279
|
-
|
|
280
|
-
//# sourceMappingURL=yaml-generator.js.map
|
|
@@ -64,10 +64,10 @@ var __webpack_exports__ = {};
|
|
|
64
64
|
const sdk_namespaceObject = require("@anthropic-ai/sdk");
|
|
65
65
|
const identity_namespaceObject = require("@azure/identity");
|
|
66
66
|
const env_namespaceObject = require("@rpascene/shared/env");
|
|
67
|
+
const types_namespaceObject = require("@rpascene/shared/env/types");
|
|
67
68
|
const img_namespaceObject = require("@rpascene/shared/img");
|
|
68
69
|
const logger_namespaceObject = require("@rpascene/shared/logger");
|
|
69
70
|
const utils_namespaceObject = require("@rpascene/shared/utils");
|
|
70
|
-
const types_namespaceObject = require("@rpascene/shared/env/types");
|
|
71
71
|
const external_https_proxy_agent_namespaceObject = require("https-proxy-agent");
|
|
72
72
|
const external_jsonrepair_namespaceObject = require("jsonrepair");
|
|
73
73
|
const external_openai_namespaceObject = require("openai");
|
|
@@ -409,8 +409,10 @@ var __webpack_exports__ = {};
|
|
|
409
409
|
};
|
|
410
410
|
return responseFormat;
|
|
411
411
|
};
|
|
412
|
-
async function callAIWithObjectResponse(messages, AIActionTypeValue, modelConfig) {
|
|
413
|
-
const response = await callAI(messages, AIActionTypeValue, modelConfig
|
|
412
|
+
async function callAIWithObjectResponse(messages, AIActionTypeValue, modelConfig, abortSignal) {
|
|
413
|
+
const response = await callAI(messages, AIActionTypeValue, modelConfig, {
|
|
414
|
+
abortSignal
|
|
415
|
+
});
|
|
414
416
|
(0, utils_namespaceObject.assert)(response, 'empty response');
|
|
415
417
|
const vlMode = modelConfig.vlMode;
|
|
416
418
|
const jsonContent = safeParseJson(response.content, vlMode);
|
|
@@ -419,8 +421,10 @@ var __webpack_exports__ = {};
|
|
|
419
421
|
usage: response.usage
|
|
420
422
|
};
|
|
421
423
|
}
|
|
422
|
-
async function callAIWithStringResponse(msgs, AIActionTypeValue, modelConfig) {
|
|
423
|
-
const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig
|
|
424
|
+
async function callAIWithStringResponse(msgs, AIActionTypeValue, modelConfig, abortSignal) {
|
|
425
|
+
const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig, {
|
|
426
|
+
abortSignal
|
|
427
|
+
});
|
|
424
428
|
return {
|
|
425
429
|
content,
|
|
426
430
|
usage
|
|
@@ -503,7 +507,8 @@ var __webpack_exports__ = {};
|
|
|
503
507
|
yht_access_token: config.yht_access_token,
|
|
504
508
|
traceId: config.traceId
|
|
505
509
|
},
|
|
506
|
-
body: JSON.stringify(requestBody)
|
|
510
|
+
body: JSON.stringify(requestBody),
|
|
511
|
+
signal: null == options ? void 0 : options.abortSignal
|
|
507
512
|
});
|
|
508
513
|
if (!response.ok) {
|
|
509
514
|
const errorText = await response.text();
|
|
@@ -630,5 +635,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
630
635
|
Object.defineProperty(exports, '__esModule', {
|
|
631
636
|
value: true
|
|
632
637
|
});
|
|
633
|
-
|
|
634
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -45,7 +45,7 @@ const pointToBbox = (point, width, height)=>[
|
|
|
45
45
|
Math.round(Math.min(point.y + bboxSize / 2, height))
|
|
46
46
|
];
|
|
47
47
|
async function uiTarsPlanning(userInstruction, options) {
|
|
48
|
-
const { conversationHistory, context, modelConfig } = options;
|
|
48
|
+
const { conversationHistory, context, modelConfig, abortSignal } = options;
|
|
49
49
|
const { uiTarsModelVersion } = modelConfig;
|
|
50
50
|
const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + userInstruction;
|
|
51
51
|
const imagePayload = await resizeImageForUiTars(context.screenshotBase64, context.size, uiTarsModelVersion);
|
|
@@ -66,7 +66,7 @@ async function uiTarsPlanning(userInstruction, options) {
|
|
|
66
66
|
content: systemPrompt
|
|
67
67
|
},
|
|
68
68
|
...conversationHistory.snapshot()
|
|
69
|
-
], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
|
|
69
|
+
], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig, abortSignal);
|
|
70
70
|
const convertedText = convertBboxToCoordinates(res.content);
|
|
71
71
|
const { size } = context;
|
|
72
72
|
const { parsed } = (0, action_parser_namespaceObject.actionParser)({
|
|
@@ -234,5 +234,3 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
|
|
|
234
234
|
Object.defineProperty(exports, '__esModule', {
|
|
235
235
|
value: true
|
|
236
236
|
});
|
|
237
|
-
|
|
238
|
-
//# sourceMappingURL=ui-tars-planning.js.map
|
package/dist/lib/device/index.js
CHANGED
package/dist/lib/image/index.js
CHANGED