@midscene/core 1.16.0 → 1.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent-options.mjs +36 -0
- package/dist/es/agent/agent-options.mjs.map +1 -0
- package/dist/es/agent/agent-report-manager.mjs +323 -0
- package/dist/es/agent/agent-report-manager.mjs.map +1 -0
- package/dist/es/agent/agent.mjs +112 -228
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/cache-config.mjs +21 -1
- package/dist/es/agent/cache-config.mjs.map +1 -1
- package/dist/es/agent/tasks.mjs +3 -1
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +1 -1
- package/dist/es/report-generator.mjs +73 -33
- package/dist/es/report-generator.mjs.map +1 -1
- package/dist/es/report-html-template.mjs +1 -1
- package/dist/es/test-runner/engine/document-runtime.mjs +30 -24
- package/dist/es/test-runner/engine/document-runtime.mjs.map +1 -1
- package/dist/es/test-runner/engine/project-runtime.mjs +17 -11
- package/dist/es/test-runner/engine/project-runtime.mjs.map +1 -1
- package/dist/es/test-runner/engine/resource-operations.mjs +33 -8
- package/dist/es/test-runner/engine/resource-operations.mjs.map +1 -1
- package/dist/es/test-runner/engine/run-collected-case.mjs +1 -0
- package/dist/es/test-runner/engine/run-collected-case.mjs.map +1 -1
- package/dist/es/test-runner/engine/run-workflow-document.mjs +3 -0
- package/dist/es/test-runner/engine/run-workflow-document.mjs.map +1 -1
- package/dist/es/test-runner/engine/setup-timeout.mjs +37 -0
- package/dist/es/test-runner/engine/setup-timeout.mjs.map +1 -0
- package/dist/es/utils.mjs +1 -1
- package/dist/es/yaml/execution-session.mjs +2 -1
- package/dist/es/yaml/execution-session.mjs.map +1 -1
- package/dist/es/yaml/legacy-yaml-report.mjs +1 -1
- package/dist/es/yaml/legacy-yaml-report.mjs.map +1 -1
- package/dist/es/yaml/player.mjs +17 -3
- package/dist/es/yaml/player.mjs.map +1 -1
- package/dist/lib/agent/agent-options.js +73 -0
- package/dist/lib/agent/agent-options.js.map +1 -0
- package/dist/lib/agent/agent-report-manager.js +357 -0
- package/dist/lib/agent/agent-report-manager.js.map +1 -0
- package/dist/lib/agent/agent.js +110 -226
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/cache-config.js +23 -0
- package/dist/lib/agent/cache-config.js.map +1 -1
- package/dist/lib/agent/tasks.js +3 -1
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/utils.js +1 -1
- package/dist/lib/report-generator.js +73 -33
- package/dist/lib/report-generator.js.map +1 -1
- package/dist/lib/report-html-template.js +1 -1
- package/dist/lib/test-runner/engine/document-runtime.js +30 -24
- package/dist/lib/test-runner/engine/document-runtime.js.map +1 -1
- package/dist/lib/test-runner/engine/project-runtime.js +17 -11
- package/dist/lib/test-runner/engine/project-runtime.js.map +1 -1
- package/dist/lib/test-runner/engine/resource-operations.js +36 -8
- package/dist/lib/test-runner/engine/resource-operations.js.map +1 -1
- package/dist/lib/test-runner/engine/run-collected-case.js +1 -0
- package/dist/lib/test-runner/engine/run-collected-case.js.map +1 -1
- package/dist/lib/test-runner/engine/run-workflow-document.js +3 -0
- package/dist/lib/test-runner/engine/run-workflow-document.js.map +1 -1
- package/dist/lib/test-runner/engine/setup-timeout.js +71 -0
- package/dist/lib/test-runner/engine/setup-timeout.js.map +1 -0
- package/dist/lib/utils.js +1 -1
- package/dist/lib/yaml/execution-session.js +4 -0
- package/dist/lib/yaml/execution-session.js.map +1 -1
- package/dist/lib/yaml/legacy-yaml-report.js +1 -1
- package/dist/lib/yaml/legacy-yaml-report.js.map +1 -1
- package/dist/lib/yaml/player.js +16 -2
- package/dist/lib/yaml/player.js.map +1 -1
- package/dist/types/agent/agent-options.d.ts +4 -0
- package/dist/types/agent/agent-options.d.ts.map +1 -0
- package/dist/types/agent/agent-report-manager.d.ts +92 -0
- package/dist/types/agent/agent-report-manager.d.ts.map +1 -0
- package/dist/types/agent/agent.d.ts +31 -24
- package/dist/types/agent/agent.d.ts.map +1 -1
- package/dist/types/agent/cache-config.d.ts +10 -0
- package/dist/types/agent/cache-config.d.ts.map +1 -1
- package/dist/types/agent/tasks.d.ts +1 -0
- package/dist/types/agent/tasks.d.ts.map +1 -1
- package/dist/types/report-generator.d.ts +5 -1
- package/dist/types/report-generator.d.ts.map +1 -1
- package/dist/types/test-runner/engine/document-runtime.d.ts.map +1 -1
- package/dist/types/test-runner/engine/project-runtime.d.ts.map +1 -1
- package/dist/types/test-runner/engine/resource-operations.d.ts +5 -0
- package/dist/types/test-runner/engine/resource-operations.d.ts.map +1 -1
- package/dist/types/test-runner/engine/run-collected-case.d.ts.map +1 -1
- package/dist/types/test-runner/engine/run-workflow-document.d.ts.map +1 -1
- package/dist/types/test-runner/engine/setup-timeout.d.ts +3 -0
- package/dist/types/test-runner/engine/setup-timeout.d.ts.map +1 -0
- package/dist/types/test-runner/engine/types.d.ts +7 -0
- package/dist/types/test-runner/engine/types.d.ts.map +1 -1
- package/dist/types/test-runner/project.d.ts +22 -0
- package/dist/types/test-runner/project.d.ts.map +1 -1
- package/dist/types/yaml/execution-session.d.ts +2 -0
- package/dist/types/yaml/execution-session.d.ts.map +1 -1
- package/dist/types/yaml/player.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/es/agent/agent.mjs
CHANGED
|
@@ -1,17 +1,11 @@
|
|
|
1
1
|
import { resolveModelForTask } from "../ai-model/models/task-routing.mjs";
|
|
2
2
|
import { INTERNAL_CALL_ID_FIELD } from "../ai-model/service-caller/index.mjs";
|
|
3
|
-
import { IS_REPORT_BUILD } from "../constants.mjs";
|
|
4
3
|
import js_yaml from "js-yaml";
|
|
5
|
-
import { generateTestRunReportScriptTag } from "../dump/html-utils.mjs";
|
|
6
4
|
import { ScreenshotItem } from "../screenshot-item.mjs";
|
|
7
5
|
import service from "../service/index.mjs";
|
|
8
|
-
import { waitForResourceIdle } from "../test-runner/engine/resource-operations.mjs";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { ExecutionDump, ReportActionDump } from "../types.mjs";
|
|
12
|
-
import { YamlExecutionOwnershipError, currentYamlSignal, enterYamlAction, enterYamlExecution, runInYamlExecutionContext } from "../yaml/execution-session.mjs";
|
|
13
|
-
import { ReportGenerator, assertReportGenerationOptions } from "../report-generator.mjs";
|
|
14
|
-
import { getVersion, processCacheConfig, reportHTMLContent } from "../utils.mjs";
|
|
6
|
+
import { hasPendingResourceOperation, waitForResourceIdle } from "../test-runner/engine/resource-operations.mjs";
|
|
7
|
+
import { ExecutionDump } from "../types.mjs";
|
|
8
|
+
import { YamlExecutionOwnershipError, currentYamlSignal, enterYamlAction, enterYamlExecution, hasActiveYamlExecution, runInYamlExecutionContext } from "../yaml/execution-session.mjs";
|
|
15
9
|
import { ScriptPlayer, buildDetailedLocateParam, buildDetailedLocateParamAndRestParams, parseYamlScript } from "../yaml/index.mjs";
|
|
16
10
|
import { readFile } from "node:fs/promises";
|
|
17
11
|
import { basename, resolve as external_node_path_resolve } from "node:path";
|
|
@@ -20,10 +14,12 @@ import { serializeError } from "@midscene/shared/agent-tools/error-formatter";
|
|
|
20
14
|
import { observationArtifactAdapterSymbol } from "@midscene/shared/agent-tools/observation-artifact";
|
|
21
15
|
import { MIDSCENE_REPLANNING_CYCLE_LIMIT, ModelConfigManager, globalConfigManager, globalModelConfigManager } from "@midscene/shared/env";
|
|
22
16
|
import { getDebug } from "@midscene/shared/logger";
|
|
23
|
-
import { assert,
|
|
17
|
+
import { assert, uuid } from "@midscene/shared/utils";
|
|
24
18
|
import { defineActionRegisterFileChooserAccept, defineActionSleep } from "../device/index.mjs";
|
|
25
19
|
import { ActionReadiness, isActionReadinessError } from "./action-readiness.mjs";
|
|
26
|
-
import {
|
|
20
|
+
import { normalizeAgentOptions, withAgentDefaults } from "./agent-options.mjs";
|
|
21
|
+
import { AgentReportManager } from "./agent-report-manager.mjs";
|
|
22
|
+
import { normalizeAgentCacheConfig } from "./cache-config.mjs";
|
|
27
23
|
import { FileChooserAccepter } from "./file-chooser.mjs";
|
|
28
24
|
import { Insight } from "./insight.mjs";
|
|
29
25
|
import { MetricsCollector } from "./metrics.mjs";
|
|
@@ -37,7 +33,7 @@ import { TaskExecutor, locatePlanForLocate, withFileChooser } from "./tasks.mjs"
|
|
|
37
33
|
import { commonAgentTestRunnerNodeDefinitions } from "./test-runner-nodes.mjs";
|
|
38
34
|
import { UIObservationImpl, UIObserverImpl, uiContextFromObservationRecord } from "./ui-observer.mjs";
|
|
39
35
|
import { locateParamStr, paramStr, taskTitleStr, typeStr } from "./ui-utils.mjs";
|
|
40
|
-
import { commonContextParser,
|
|
36
|
+
import { commonContextParser, normalizeFilePaths, normalizeScrollType } from "./utils.mjs";
|
|
41
37
|
function _define_property(obj, key, value) {
|
|
42
38
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
43
39
|
value: value,
|
|
@@ -67,66 +63,38 @@ class Agent {
|
|
|
67
63
|
static getTestRunnerNodeDefinitions() {
|
|
68
64
|
return commonAgentTestRunnerNodeDefinitions;
|
|
69
65
|
}
|
|
66
|
+
get dump() {
|
|
67
|
+
return this.reportManager.dump;
|
|
68
|
+
}
|
|
69
|
+
set dump(value) {
|
|
70
|
+
this.reportManager.dump = value;
|
|
71
|
+
}
|
|
72
|
+
get reportFile() {
|
|
73
|
+
return this.reportManager.reportFile;
|
|
74
|
+
}
|
|
75
|
+
set reportFile(value) {
|
|
76
|
+
this.reportManager.reportFile = value;
|
|
77
|
+
}
|
|
78
|
+
get reportFileName() {
|
|
79
|
+
return this.reportManager.reportFileName;
|
|
80
|
+
}
|
|
81
|
+
set reportFileName(value) {
|
|
82
|
+
this.reportManager.reportFileName = value;
|
|
83
|
+
}
|
|
70
84
|
_prepareForTestRunner() {
|
|
71
|
-
this.
|
|
85
|
+
this.reportManager._prepareForTestRunner();
|
|
72
86
|
}
|
|
73
87
|
async _createReportSource(scopeId, executionIds) {
|
|
74
|
-
return this.
|
|
88
|
+
return this.reportManager._createReportSource(scopeId, executionIds);
|
|
75
89
|
}
|
|
76
90
|
async _writeYamlExecutionReport(record) {
|
|
77
|
-
|
|
78
|
-
const recordIndex = existingIndex < 0 ? this.yamlExecutionRecords.length : existingIndex;
|
|
79
|
-
this.yamlExecutionRecords[recordIndex] = record;
|
|
80
|
-
try {
|
|
81
|
-
const path = await this.reportGenerator.writeRunnerReport?.(this.buildYamlReportInput(), this.yamlReportScopeId, this.getReportMeta());
|
|
82
|
-
this.reportFile = path ?? this.reportFile;
|
|
83
|
-
return path;
|
|
84
|
-
} catch (error) {
|
|
85
|
-
this.yamlExecutionRecords[recordIndex] = Object.freeze({
|
|
86
|
-
...record,
|
|
87
|
-
status: 'failed',
|
|
88
|
-
reportError: error
|
|
89
|
-
});
|
|
90
|
-
throw error;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
buildYamlReportInput() {
|
|
94
|
-
const scopedRecords = this.yamlExecutionRecords.map((entry)=>({
|
|
95
|
-
...entry,
|
|
96
|
-
...entry.execution ? {
|
|
97
|
-
execution: {
|
|
98
|
-
document: {
|
|
99
|
-
...entry.execution.document,
|
|
100
|
-
reportScopeId: this.yamlReportScopeId
|
|
101
|
-
},
|
|
102
|
-
cases: entry.execution.cases.map((outcome)=>({
|
|
103
|
-
...outcome,
|
|
104
|
-
...outcome.run ? {
|
|
105
|
-
run: {
|
|
106
|
-
...outcome.run,
|
|
107
|
-
reportScopeId: this.yamlReportScopeId
|
|
108
|
-
}
|
|
109
|
-
} : {},
|
|
110
|
-
...outcome.attempts ? {
|
|
111
|
-
attempts: outcome.attempts.map((attempt)=>({
|
|
112
|
-
...attempt,
|
|
113
|
-
reportScopeId: this.yamlReportScopeId
|
|
114
|
-
}))
|
|
115
|
-
} : {}
|
|
116
|
-
}))
|
|
117
|
-
}
|
|
118
|
-
} : {}
|
|
119
|
-
}));
|
|
120
|
-
return executionRecordsToReportInput(scopedRecords, {
|
|
121
|
-
runId: this.yamlReportScopeId
|
|
122
|
-
});
|
|
91
|
+
return this.reportManager._writeYamlExecutionReport(record);
|
|
123
92
|
}
|
|
124
93
|
get onDumpUpdate() {
|
|
125
|
-
return this.
|
|
94
|
+
return this.reportManager.onDumpUpdate;
|
|
126
95
|
}
|
|
127
96
|
set onDumpUpdate(callback) {
|
|
128
|
-
this.
|
|
129
|
-
if (callback) this.dumpUpdateListeners.push(callback);
|
|
97
|
+
this.reportManager.onDumpUpdate = callback;
|
|
130
98
|
}
|
|
131
99
|
get aiContexts() {
|
|
132
100
|
if (!this.opts.aiContexts) this.opts.aiContexts = {};
|
|
@@ -171,7 +139,9 @@ class Agent {
|
|
|
171
139
|
};
|
|
172
140
|
}
|
|
173
141
|
createInsight(getUIContext) {
|
|
174
|
-
return new Insight(
|
|
142
|
+
return new Insight({
|
|
143
|
+
createTypeQueryExecution: (...args)=>this.withReportOperation(()=>this.taskExecutor.createTypeQueryExecution(...args))
|
|
144
|
+
}, (task)=>this.resolveModelRuntime('insight', task), (apiName, callContext)=>this.resolveUserContext(apiName, callContext), getUIContext);
|
|
175
145
|
}
|
|
176
146
|
async getActionSpace() {
|
|
177
147
|
return this.fullActionSpace;
|
|
@@ -204,6 +174,7 @@ class Agent {
|
|
|
204
174
|
return await this.getUIContext('locate');
|
|
205
175
|
}
|
|
206
176
|
async startObserving(opt) {
|
|
177
|
+
this.assertReportWritable();
|
|
207
178
|
assert(!this.frozenUIContext, 'startObserving() cannot be used while the UI context is frozen (call unfreezePageContext() first)');
|
|
208
179
|
assert(!this.activeObserver, "An observation window is already active on this agent. Stop the existing observer first (await observer.stop()) before starting a new one.");
|
|
209
180
|
const observer = new UIObserverImpl({
|
|
@@ -251,31 +222,20 @@ class Agent {
|
|
|
251
222
|
this.opts.aiActionContext = context;
|
|
252
223
|
}
|
|
253
224
|
}
|
|
254
|
-
|
|
255
|
-
this.
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
return this.
|
|
225
|
+
assertReportWritable() {
|
|
226
|
+
this.reportManager.assertReportWritable();
|
|
227
|
+
}
|
|
228
|
+
async withReportOperation(operation) {
|
|
229
|
+
return this.reportManager.withReportOperation(operation);
|
|
230
|
+
}
|
|
231
|
+
async resetDump() {
|
|
232
|
+
return this.reportManager.resetDump();
|
|
233
|
+
}
|
|
234
|
+
async finalizeReport() {
|
|
235
|
+
return this.reportManager.finalizeReport();
|
|
265
236
|
}
|
|
266
237
|
appendExecutionDump(execution, runner) {
|
|
267
|
-
|
|
268
|
-
if (runner) {
|
|
269
|
-
const existingIndex = this.executionDumpIndexByRunner.get(runner);
|
|
270
|
-
if (void 0 !== existingIndex) {
|
|
271
|
-
currentDump.executions[existingIndex] = execution;
|
|
272
|
-
return;
|
|
273
|
-
}
|
|
274
|
-
currentDump.executions.push(execution);
|
|
275
|
-
this.executionDumpIndexByRunner.set(runner, currentDump.executions.length - 1);
|
|
276
|
-
return;
|
|
277
|
-
}
|
|
278
|
-
currentDump.executions.push(execution);
|
|
238
|
+
this.reportManager.appendExecutionDump(execution, runner);
|
|
279
239
|
}
|
|
280
240
|
collectUsageMetrics(execution) {
|
|
281
241
|
for (const task of execution.tasks){
|
|
@@ -300,50 +260,16 @@ class Agent {
|
|
|
300
260
|
return this.metricsCollector.snapshot();
|
|
301
261
|
}
|
|
302
262
|
dumpDataString(opt) {
|
|
303
|
-
|
|
304
|
-
this.dump.groupDescription = this.opts.groupDescription;
|
|
305
|
-
if (ifInBrowser || opt?.inlineScreenshots) return this.dump.serializeWithInlineScreenshots();
|
|
306
|
-
return this.dump.serialize();
|
|
263
|
+
return this.reportManager.dumpDataString(opt);
|
|
307
264
|
}
|
|
308
265
|
reportHTMLString(opt) {
|
|
309
|
-
|
|
310
|
-
const dumpString = this.dumpDataString(opt);
|
|
311
|
-
if (!this.yamlExecutionRecords.length) return reportHTMLContent(dumpString);
|
|
312
|
-
const reportId = this.yamlReportScopeId;
|
|
313
|
-
const runnerDump = buildAgentTestRunReportDump(this.buildYamlReportInput(), this.dump, {
|
|
314
|
-
reportId,
|
|
315
|
-
scopeId: this.yamlReportScopeId,
|
|
316
|
-
sourcePath: this.reportFile ?? '<inline-agent-report>'
|
|
317
|
-
});
|
|
318
|
-
return `${reportHTMLContent({
|
|
319
|
-
dumpString,
|
|
320
|
-
attributes: {
|
|
321
|
-
'data-group-id': reportId,
|
|
322
|
-
'data-report-id': reportId
|
|
323
|
-
}
|
|
324
|
-
})}\n${generateTestRunReportScriptTag(runnerDump)}`;
|
|
266
|
+
return this.reportManager.reportHTMLString(()=>this.dumpDataString(opt));
|
|
325
267
|
}
|
|
326
268
|
writeOutActionDumps(executionDump) {
|
|
327
|
-
|
|
328
|
-
if (exec) {
|
|
329
|
-
this.lastExecutionDump = exec;
|
|
330
|
-
this.reportGenerator.onExecutionUpdate(exec, this.getReportMeta(), this.opts.reportAttributes);
|
|
331
|
-
}
|
|
332
|
-
this.reportFile = this.reportGenerator.getReportPath();
|
|
269
|
+
this.reportManager.writeOutActionDumps(executionDump);
|
|
333
270
|
}
|
|
334
271
|
async flushReport() {
|
|
335
|
-
this.
|
|
336
|
-
await this.reportGenerator.flush();
|
|
337
|
-
return this.lastExecutionDump ? this.reportFile ?? void 0 : void 0;
|
|
338
|
-
}
|
|
339
|
-
getReportMeta() {
|
|
340
|
-
return {
|
|
341
|
-
groupName: this.dump.groupName,
|
|
342
|
-
groupDescription: this.dump.groupDescription,
|
|
343
|
-
sdkVersion: this.dump.sdkVersion,
|
|
344
|
-
modelBriefs: this.dump.modelBriefs,
|
|
345
|
-
deviceType: this.dump.deviceType
|
|
346
|
-
};
|
|
272
|
+
return this.reportManager.flushReport();
|
|
347
273
|
}
|
|
348
274
|
async callbackOnTaskStartTip(task) {
|
|
349
275
|
const param = paramStr(task);
|
|
@@ -354,6 +280,9 @@ class Agent {
|
|
|
354
280
|
return async (param)=>await this.callActionInActionSpace(name, param);
|
|
355
281
|
}
|
|
356
282
|
async callActionInActionSpace(type, opt, abortSignal) {
|
|
283
|
+
return this.withReportOperation(()=>this.executeActionInActionSpace(type, opt, abortSignal));
|
|
284
|
+
}
|
|
285
|
+
async executeActionInActionSpace(type, opt, abortSignal) {
|
|
357
286
|
debug('callActionInActionSpace', type, ',', opt);
|
|
358
287
|
const clonedActionParam = opt && 'object' == typeof opt && !Array.isArray(opt) ? {
|
|
359
288
|
...opt
|
|
@@ -376,12 +305,14 @@ class Agent {
|
|
|
376
305
|
return output;
|
|
377
306
|
}
|
|
378
307
|
async aiTap(locatePrompt, opt) {
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
await this.
|
|
384
|
-
|
|
308
|
+
await this.withReportOperation(async ()=>{
|
|
309
|
+
assert(locatePrompt, 'missing locate prompt for tap');
|
|
310
|
+
const detailedLocateParam = buildDetailedLocateParam(locatePrompt, this.withContext('aiTap', opt));
|
|
311
|
+
const fileChooserAccept = opt?.fileChooserAccept ? this.normalizeFileInput(opt.fileChooserAccept) : void 0;
|
|
312
|
+
await withFileChooser(this.interface, fileChooserAccept, async ()=>{
|
|
313
|
+
await this.callActionInActionSpace('Tap', {
|
|
314
|
+
locate: detailedLocateParam
|
|
315
|
+
});
|
|
385
316
|
});
|
|
386
317
|
});
|
|
387
318
|
}
|
|
@@ -512,6 +443,9 @@ class Agent {
|
|
|
512
443
|
});
|
|
513
444
|
}
|
|
514
445
|
async aiAct(taskPrompt, opt) {
|
|
446
|
+
return this.withReportOperation(()=>this.executeAiAct(taskPrompt, opt));
|
|
447
|
+
}
|
|
448
|
+
async executeAiAct(taskPrompt, opt) {
|
|
515
449
|
const internalOptions = opt;
|
|
516
450
|
const internalReportDisplay = internalOptions?._internalReportDisplay;
|
|
517
451
|
const taskPromptText = 'string' == typeof taskPrompt ? taskPrompt : taskPrompt.prompt;
|
|
@@ -619,6 +553,9 @@ class Agent {
|
|
|
619
553
|
return result;
|
|
620
554
|
}
|
|
621
555
|
async runMarkdown(markdownPath, opt) {
|
|
556
|
+
return this.withReportOperation(()=>this.executeMarkdown(markdownPath, opt));
|
|
557
|
+
}
|
|
558
|
+
async executeMarkdown(markdownPath, opt) {
|
|
622
559
|
const markdown = await readFile(markdownPath, 'utf-8');
|
|
623
560
|
const { prompt } = await markdownToAiActPrompt(markdown, markdownPath);
|
|
624
561
|
return this.aiAct(prompt, {
|
|
@@ -630,7 +567,7 @@ class Agent {
|
|
|
630
567
|
});
|
|
631
568
|
}
|
|
632
569
|
async runGherkinScenario(scenarioText, opt) {
|
|
633
|
-
return runGherkinScenario(this, scenarioText, opt);
|
|
570
|
+
return this.withReportOperation(()=>runGherkinScenario(this, scenarioText, opt));
|
|
634
571
|
}
|
|
635
572
|
async aiAction(taskPrompt, opt) {
|
|
636
573
|
return this.aiAct(taskPrompt, opt);
|
|
@@ -651,6 +588,9 @@ class Agent {
|
|
|
651
588
|
return this.createInsight().aiAsk(prompt, opt);
|
|
652
589
|
}
|
|
653
590
|
async aiLocate(prompt, opt) {
|
|
591
|
+
return this.withReportOperation(()=>this.executeLocate(prompt, opt));
|
|
592
|
+
}
|
|
593
|
+
async executeLocate(prompt, opt) {
|
|
654
594
|
const locateParam = buildDetailedLocateParam(prompt, this.withContext('aiLocate', opt));
|
|
655
595
|
assert(locateParam, 'cannot get locate param for aiLocate');
|
|
656
596
|
const locatePlan = locatePlanForLocate(locateParam);
|
|
@@ -680,16 +620,19 @@ class Agent {
|
|
|
680
620
|
async aiWaitFor(assertion, opt) {
|
|
681
621
|
const modelRuntime = this.resolveModelRuntime('insight', 'boolean');
|
|
682
622
|
const options = this.withContext('aiWaitFor', opt);
|
|
683
|
-
await this.taskExecutor.waitFor(assertion, {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
623
|
+
await this.withReportOperation(()=>this.taskExecutor.waitFor(assertion, {
|
|
624
|
+
...options,
|
|
625
|
+
timeoutMs: options?.timeoutMs || 15000,
|
|
626
|
+
checkIntervalMs: options?.checkIntervalMs || 3000
|
|
627
|
+
}, modelRuntime));
|
|
688
628
|
}
|
|
689
629
|
async ai(...args) {
|
|
690
630
|
return this.aiAct(...args);
|
|
691
631
|
}
|
|
692
632
|
async runYaml(yamlScriptContent, options) {
|
|
633
|
+
return this.withReportOperation(()=>this.executeYaml(yamlScriptContent, options));
|
|
634
|
+
}
|
|
635
|
+
async executeYaml(yamlScriptContent, options) {
|
|
693
636
|
if (this.destroyed) throw new Error('Cannot run YAML using a destroyed Agent.');
|
|
694
637
|
const script = parseYamlScript(yamlScriptContent, 'yaml');
|
|
695
638
|
const player = new ScriptPlayer(script, async ()=>({
|
|
@@ -709,20 +652,16 @@ class Agent {
|
|
|
709
652
|
}
|
|
710
653
|
async evaluateJavaScript(script) {
|
|
711
654
|
assert(this.interface.evaluateJavaScript, 'evaluateJavaScript is not supported in current agent');
|
|
712
|
-
return this.interface.evaluateJavaScript(script);
|
|
655
|
+
return this.withReportOperation(()=>this.interface.evaluateJavaScript(script));
|
|
713
656
|
}
|
|
714
657
|
addDumpUpdateListener(listener) {
|
|
715
|
-
this.
|
|
716
|
-
return ()=>{
|
|
717
|
-
this.removeDumpUpdateListener(listener);
|
|
718
|
-
};
|
|
658
|
+
return this.reportManager.addDumpUpdateListener(listener);
|
|
719
659
|
}
|
|
720
660
|
removeDumpUpdateListener(listener) {
|
|
721
|
-
|
|
722
|
-
if (index > -1) this.dumpUpdateListeners.splice(index, 1);
|
|
661
|
+
this.reportManager.removeDumpUpdateListener(listener);
|
|
723
662
|
}
|
|
724
663
|
clearDumpUpdateListeners() {
|
|
725
|
-
this.
|
|
664
|
+
this.reportManager.clearDumpUpdateListeners();
|
|
726
665
|
}
|
|
727
666
|
addProgressListener(listener) {
|
|
728
667
|
return this.progressBus.subscribe(listener);
|
|
@@ -734,17 +673,13 @@ class Agent {
|
|
|
734
673
|
this.progressBus.clear();
|
|
735
674
|
}
|
|
736
675
|
notifyDumpUpdateListeners(executionDump) {
|
|
737
|
-
|
|
738
|
-
for (const listener of this.dumpUpdateListeners)try {
|
|
739
|
-
listener(dumpString, executionDump);
|
|
740
|
-
} catch (error) {
|
|
741
|
-
console.error('Error in onDumpUpdate listener', error);
|
|
742
|
-
}
|
|
676
|
+
this.reportManager.notifyDumpUpdateListeners(executionDump);
|
|
743
677
|
}
|
|
744
678
|
async destroy() {
|
|
745
679
|
if (this.destroyed) return;
|
|
746
680
|
this.destroyed = true;
|
|
747
681
|
await waitForResourceIdle(this);
|
|
682
|
+
await this.reportManager.waitForOperations();
|
|
748
683
|
for (const observer of this.ownedObservers)try {
|
|
749
684
|
await observer.dispose();
|
|
750
685
|
} catch (error) {
|
|
@@ -758,17 +693,18 @@ class Agent {
|
|
|
758
693
|
} catch (error) {
|
|
759
694
|
interfaceDestroyError = error;
|
|
760
695
|
}
|
|
761
|
-
await this.
|
|
762
|
-
|
|
763
|
-
this.reportFile = finalPath;
|
|
764
|
-
this.resetDump();
|
|
696
|
+
await this.reportManager.waitForReset();
|
|
697
|
+
if (!hasActiveYamlExecution(this)) await this.finalizeReport();
|
|
765
698
|
if (interfaceDestroyError) throw interfaceDestroyError;
|
|
766
699
|
}
|
|
767
700
|
async sleep(ms, options) {
|
|
768
701
|
options?.abortSignal?.throwIfAborted();
|
|
769
|
-
await this.taskExecutor.sleep(ms, options?.abortSignal);
|
|
702
|
+
await this.withReportOperation(()=>this.taskExecutor.sleep(ms, options?.abortSignal));
|
|
770
703
|
}
|
|
771
704
|
async recordToReport(title, opt) {
|
|
705
|
+
return this.withReportOperation(()=>this.writeRecordToReport(title, opt));
|
|
706
|
+
}
|
|
707
|
+
async writeRecordToReport(title, opt) {
|
|
772
708
|
const now = Date.now();
|
|
773
709
|
const screenshots = opt?.screenshots;
|
|
774
710
|
const screenshotBase64 = opt?.screenshotBase64;
|
|
@@ -826,10 +762,13 @@ class Agent {
|
|
|
826
762
|
});
|
|
827
763
|
this.appendExecutionDump(executionDump);
|
|
828
764
|
this.writeOutActionDumps(executionDump);
|
|
829
|
-
await this.
|
|
765
|
+
await this.reportManager.flushPendingWrites();
|
|
830
766
|
this.notifyDumpUpdateListeners(executionDump);
|
|
831
767
|
}
|
|
832
768
|
async recordErrorToReport(title, opt) {
|
|
769
|
+
return this.withReportOperation(()=>this.writeErrorToReport(title, opt));
|
|
770
|
+
}
|
|
771
|
+
async writeErrorToReport(title, opt) {
|
|
833
772
|
const now = Date.now();
|
|
834
773
|
const error = serializeError(opt.error);
|
|
835
774
|
const recorder = [];
|
|
@@ -869,7 +808,7 @@ class Agent {
|
|
|
869
808
|
});
|
|
870
809
|
this.appendExecutionDump(executionDump);
|
|
871
810
|
this.writeOutActionDumps(executionDump);
|
|
872
|
-
await this.
|
|
811
|
+
await this.reportManager.flushPendingWrites();
|
|
873
812
|
this.notifyDumpUpdateListeners(executionDump);
|
|
874
813
|
}
|
|
875
814
|
async logScreenshot(title, opt) {
|
|
@@ -895,25 +834,6 @@ class Agent {
|
|
|
895
834
|
this.frozenUIContext = void 0;
|
|
896
835
|
debug('Page context unfrozen successfully');
|
|
897
836
|
}
|
|
898
|
-
processCacheConfig(opts) {
|
|
899
|
-
validateAgentCacheInput(opts.cache);
|
|
900
|
-
const cacheConfig = processCacheConfig(opts.cache, opts.cacheId || 'default');
|
|
901
|
-
if (!cacheConfig) return null;
|
|
902
|
-
if ('object' == typeof cacheConfig && null !== cacheConfig) {
|
|
903
|
-
const id = cacheConfig.id;
|
|
904
|
-
const strategyValue = cacheConfig.strategy ?? 'read-write';
|
|
905
|
-
const isReadOnly = 'read-only' === strategyValue;
|
|
906
|
-
const isWriteOnly = 'write-only' === strategyValue;
|
|
907
|
-
return {
|
|
908
|
-
id,
|
|
909
|
-
enabled: !isWriteOnly,
|
|
910
|
-
readOnly: isReadOnly,
|
|
911
|
-
writeOnly: isWriteOnly,
|
|
912
|
-
cacheDir: cacheConfig.cacheDir?.trim()
|
|
913
|
-
};
|
|
914
|
-
}
|
|
915
|
-
return null;
|
|
916
|
-
}
|
|
917
837
|
normalizeFileInput(files) {
|
|
918
838
|
const filesArray = Array.isArray(files) ? files : [
|
|
919
839
|
files
|
|
@@ -927,11 +847,7 @@ class Agent {
|
|
|
927
847
|
constructor(interfaceInstance, opts){
|
|
928
848
|
_define_property(this, "interface", void 0);
|
|
929
849
|
_define_property(this, "service", void 0);
|
|
930
|
-
_define_property(this, "
|
|
931
|
-
_define_property(this, "reportFile", void 0);
|
|
932
|
-
_define_property(this, "reportFileName", void 0);
|
|
933
|
-
_define_property(this, "yamlReportScopeId", uuid());
|
|
934
|
-
_define_property(this, "yamlExecutionRecords", []);
|
|
850
|
+
_define_property(this, "reportManager", void 0);
|
|
935
851
|
_define_property(this, "taskExecutor", void 0);
|
|
936
852
|
_define_property(this, "opts", void 0);
|
|
937
853
|
_define_property(this, "dryMode", false);
|
|
@@ -940,7 +856,6 @@ class Agent {
|
|
|
940
856
|
_define_property(this, "metricsCollector", new MetricsCollector());
|
|
941
857
|
_define_property(this, "usageCallCounter", 0);
|
|
942
858
|
_define_property(this, "countedUsageKeys", new Set());
|
|
943
|
-
_define_property(this, "dumpUpdateListeners", []);
|
|
944
859
|
_define_property(this, "progressBus", new AgentProgressBus());
|
|
945
860
|
_define_property(this, "destroyed", false);
|
|
946
861
|
_define_property(this, "actionReadiness", void 0);
|
|
@@ -948,44 +863,17 @@ class Agent {
|
|
|
948
863
|
_define_property(this, "frozenUIContext", void 0);
|
|
949
864
|
_define_property(this, "activeObserver", null);
|
|
950
865
|
_define_property(this, "ownedObservers", new Set());
|
|
951
|
-
_define_property(this, "executionDumpIndexByRunner", new WeakMap());
|
|
952
866
|
_define_property(this, "fullActionSpace", void 0);
|
|
953
867
|
_define_property(this, "activeFileChooserAccepter", void 0);
|
|
954
868
|
_define_property(this, "activeFileChooserAllowedDir", void 0);
|
|
955
|
-
_define_property(this, "reportGenerator", void 0);
|
|
956
|
-
_define_property(this, "lastExecutionDump", void 0);
|
|
957
869
|
this.interface = interfaceInstance;
|
|
958
|
-
this.opts =
|
|
959
|
-
generateReport: true,
|
|
960
|
-
persistExecutionDump: false,
|
|
961
|
-
autoPrintReportMsg: true,
|
|
962
|
-
groupName: 'Midscene Report',
|
|
963
|
-
groupDescription: ''
|
|
964
|
-
}, opts || {});
|
|
965
|
-
assertReportGenerationOptions(this.opts);
|
|
870
|
+
this.opts = withAgentDefaults(opts);
|
|
966
871
|
if (void 0 !== this.opts.waitForActionReady) this.actionReadiness = new ActionReadiness(this.opts.waitForActionReady);
|
|
967
|
-
|
|
968
|
-
for (const [key, value] of Object.entries(this.opts.aiContexts ?? {})){
|
|
969
|
-
if (!isAgentAIContextKey(key)) throw new TypeError(`Unknown Agent context key: ${key}`);
|
|
970
|
-
if (void 0 !== value && 'string' != typeof value) throw new TypeError(`Agent context "${key}" must be a string`);
|
|
971
|
-
}
|
|
972
|
-
const deprecatedAiActContextOption = void 0 !== this.opts.aiActContext ? 'aiActContext' : void 0 !== this.opts.aiActionContext ? 'aiActionContext' : void 0;
|
|
973
|
-
if (deprecatedAiActContextOption) warn(`Agent option "${deprecatedAiActContextOption}" is deprecated; use "aiContexts.aiAct" instead. When both are provided, "aiContexts.aiAct" takes precedence.`);
|
|
974
|
-
const normalizedAIContexts = {
|
|
975
|
-
...this.opts.aiContexts
|
|
976
|
-
};
|
|
977
|
-
const resolvedAiActContext = normalizedAIContexts.aiAct ?? this.opts.aiActContext ?? this.opts.aiActionContext;
|
|
978
|
-
if (void 0 !== resolvedAiActContext) {
|
|
979
|
-
normalizedAIContexts.aiAct = resolvedAiActContext;
|
|
980
|
-
this.opts.aiActContext = resolvedAiActContext;
|
|
981
|
-
this.opts.aiActionContext = resolvedAiActContext;
|
|
982
|
-
}
|
|
983
|
-
this.opts.aiContexts = normalizedAIContexts;
|
|
984
|
-
if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
|
|
872
|
+
normalizeAgentOptions(this.opts, warn);
|
|
985
873
|
this.modelConfigManager = opts?.modelConfig ? new ModelConfigManager(opts.modelConfig, opts.createOpenAIClient) : opts?.createOpenAIClient ? new AgentScopedModelConfigManager(globalModelConfigManager, opts.createOpenAIClient) : globalModelConfigManager;
|
|
986
874
|
this.onTaskStartTip = this.opts.onTaskStartTip;
|
|
987
875
|
this.service = new service(async ()=>this.getUIContext());
|
|
988
|
-
const cacheConfigObj =
|
|
876
|
+
const cacheConfigObj = normalizeAgentCacheConfig(opts || {});
|
|
989
877
|
if (cacheConfigObj) this.taskCache = new TaskCache(cacheConfigObj.id, cacheConfigObj.enabled, void 0, {
|
|
990
878
|
readOnly: cacheConfigObj.readOnly,
|
|
991
879
|
writeOnly: cacheConfigObj.writeOnly,
|
|
@@ -1016,30 +904,26 @@ class Agent {
|
|
|
1016
904
|
operationResource: this,
|
|
1017
905
|
actionSpace: this.fullActionSpace,
|
|
1018
906
|
hooks: {
|
|
907
|
+
onExecutionCreated: (runner)=>{
|
|
908
|
+
this.reportManager.bindRunner(runner);
|
|
909
|
+
},
|
|
1019
910
|
onSnapshotChange: async (runner)=>{
|
|
911
|
+
this.reportManager.assertRunnerCurrent(runner);
|
|
1020
912
|
const executionDump = runner.dump();
|
|
1021
913
|
this.appendExecutionDump(executionDump, runner);
|
|
1022
914
|
this.collectUsageMetrics(executionDump);
|
|
1023
915
|
this.writeOutActionDumps(executionDump);
|
|
1024
|
-
await this.
|
|
1025
|
-
|
|
1026
|
-
for (const listener of this.dumpUpdateListeners)try {
|
|
1027
|
-
listener(dumpString, executionDump);
|
|
1028
|
-
} catch (error) {
|
|
1029
|
-
console.error('Error in onDumpUpdate listener', error);
|
|
1030
|
-
}
|
|
916
|
+
await this.reportManager.flushPendingWrites();
|
|
917
|
+
this.reportManager.notifyDumpUpdateListeners(executionDump);
|
|
1031
918
|
},
|
|
1032
919
|
onProgress: this.progressBus.publish
|
|
1033
920
|
}
|
|
1034
921
|
});
|
|
1035
|
-
this.
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
outputFormat: this.opts.outputFormat,
|
|
1041
|
-
autoPrintReportMsg: this.opts.autoPrintReportMsg,
|
|
1042
|
-
reuseExistingReport: this.opts.reportAttributes?.['data-group-id'] === this.reportFileName
|
|
922
|
+
this.reportManager = new AgentReportManager({
|
|
923
|
+
getOptions: ()=>this.opts,
|
|
924
|
+
getDeviceType: ()=>this.interface.interfaceType,
|
|
925
|
+
isDestroyed: ()=>this.destroyed,
|
|
926
|
+
isExternallyBusy: ()=>Boolean(this.activeObserver) || hasActiveYamlExecution(this) || hasPendingResourceOperation(this)
|
|
1043
927
|
});
|
|
1044
928
|
Object.defineProperty(this, observationArtifactAdapterSymbol, {
|
|
1045
929
|
value: {
|