@ui-tars-test/cli 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands.js +1 -1
- package/dist/cli/commands.js.map +1 -1
- package/dist/cli/commands.mjs +1 -1
- package/dist/cli/commands.mjs.map +1 -1
- package/dist/cli/start.js +371 -77
- package/dist/cli/start.js.map +1 -1
- package/dist/cli/start.mjs +369 -77
- package/dist/cli/start.mjs.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/src/cli/commands.d.ts.map +1 -1
- package/dist/src/cli/start.d.ts +1 -0
- package/dist/src/cli/start.d.ts.map +1 -1
- package/package.json +5 -5
package/dist/cli/start.mjs
CHANGED
|
@@ -5,14 +5,72 @@
|
|
|
5
5
|
import node_fs from "node:fs";
|
|
6
6
|
import node_path from "node:path";
|
|
7
7
|
import node_os from "node:os";
|
|
8
|
+
import fluent_ffmpeg from "fluent-ffmpeg";
|
|
9
|
+
import ffmpeg_static from "ffmpeg-static";
|
|
8
10
|
import node_fetch from "node-fetch";
|
|
9
|
-
import { GUIAgent } from "@
|
|
11
|
+
import { GUIAgent } from "@ui-tars-test/agent-sdk";
|
|
10
12
|
import { cancel, group, select as prompts_select, text as prompts_text } from "@clack/prompts";
|
|
11
13
|
import js_yaml from "js-yaml";
|
|
12
|
-
import { NutJSOperator } from "@
|
|
13
|
-
import { AdbOperator } from "@
|
|
14
|
-
import { BrowserOperator } from "@
|
|
14
|
+
import { NutJSOperator } from "@ui-tars-test/operator-nutjs";
|
|
15
|
+
import { AdbOperator } from "@ui-tars-test/operator-adb";
|
|
16
|
+
import { BrowserOperator } from "@ui-tars-test/operator-browser";
|
|
17
|
+
const saveConversationLog = (events, targetOutputDir, sessionId)=>{
|
|
18
|
+
const logPath = node_path.join(targetOutputDir, `${sessionId}.md`);
|
|
19
|
+
let content = `# Conversation Log - ${sessionId}\n\n`;
|
|
20
|
+
events.forEach((e)=>{
|
|
21
|
+
const time = new Date(e.timestamp || Date.now()).toISOString();
|
|
22
|
+
content += `## [${time}] ${e.type}\n`;
|
|
23
|
+
if (e.content) if (Array.isArray(e.content)) e.content.forEach((part)=>{
|
|
24
|
+
if ('text' === part.type) content += `${part.text}\n`;
|
|
25
|
+
if ('image_url' === part.type) content += `[Image Content]\n`;
|
|
26
|
+
});
|
|
27
|
+
else if ('string' == typeof e.content) content += `${e.content}\n`;
|
|
28
|
+
else content += `\`\`\`json\n${JSON.stringify(e.content, null, 2)}\n\`\`\`\n`;
|
|
29
|
+
if ('tool_call' === e.type) {
|
|
30
|
+
if (e.name) content += `> Tool Call: ${e.name}\n`;
|
|
31
|
+
if (e.arguments) content += `> Arguments: ${JSON.stringify(e.arguments, null, 2)}\n`;
|
|
32
|
+
const toolCall = e.toolCall;
|
|
33
|
+
if (toolCall) {
|
|
34
|
+
if (toolCall.name) content += `> Tool Call: ${toolCall.name}\n`;
|
|
35
|
+
if (toolCall.arguments) content += `> Arguments: ${JSON.stringify(toolCall.arguments, null, 2)}\n`;
|
|
36
|
+
}
|
|
37
|
+
} else if ('assistant_message' === e.type) {
|
|
38
|
+
if (e.content) content += `${e.content}\n`;
|
|
39
|
+
if (e.rawContent) content += `\n> Raw Content (Debug): ${JSON.stringify(e.rawContent)}\n`;
|
|
40
|
+
if (e.toolCalls) content += `> Tool Calls: ${JSON.stringify(e.toolCalls, null, 2)}\n`;
|
|
41
|
+
const message = e.message;
|
|
42
|
+
if (message) {
|
|
43
|
+
if (message.content) content += `${message.content}\n`;
|
|
44
|
+
if (message.rawContent) content += `\n> Raw Content (Debug): ${JSON.stringify(message.rawContent)}\n`;
|
|
45
|
+
if (message.tool_calls) content += `> Tool Calls: ${JSON.stringify(message.tool_calls, null, 2)}\n`;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (e.metadata) if ('screenshot' === e.metadata.type) content += `> Action: Screenshot captured\n`;
|
|
49
|
+
else content += `> Metadata: ${JSON.stringify(e.metadata)}\n`;
|
|
50
|
+
if (e.input) content += `> Input: ${JSON.stringify(e.input, null, 2)}\n`;
|
|
51
|
+
if (e.output) content += `> Output: ${JSON.stringify(e.output, null, 2)}\n`;
|
|
52
|
+
content += '\n';
|
|
53
|
+
});
|
|
54
|
+
try {
|
|
55
|
+
node_fs.writeFileSync(logPath, content);
|
|
56
|
+
console.log(`[CLI] Conversation log saved: ${logPath}`);
|
|
57
|
+
return logPath;
|
|
58
|
+
} catch (err) {
|
|
59
|
+
console.warn(`[CLI] Failed to save conversation log: ${err}`);
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
};
|
|
15
63
|
const start = async (options)=>{
|
|
64
|
+
if (ffmpeg_static) {
|
|
65
|
+
let finalFfmpegPath = ffmpeg_static;
|
|
66
|
+
const bundledFfmpegPath = node_path.join(__dirname, 'ffmpeg');
|
|
67
|
+
if (node_fs.existsSync(bundledFfmpegPath)) {
|
|
68
|
+
finalFfmpegPath = bundledFfmpegPath;
|
|
69
|
+
console.log(`[CLI] Using bundled ffmpeg: ${finalFfmpegPath}`);
|
|
70
|
+
} else console.log(`[CLI] Using default ffmpeg-static path: ${finalFfmpegPath}`);
|
|
71
|
+
if ('darwin' === node_os.platform() && 'arm64' === node_os.arch()) console.log(`[CLI] Setting ffmpeg path: ${finalFfmpegPath}`);
|
|
72
|
+
fluent_ffmpeg.setFfmpegPath(finalFfmpegPath);
|
|
73
|
+
} else console.warn('[CLI] ffmpeg-static not found. Video generation might fail if ffmpeg is not in PATH.');
|
|
16
74
|
const CONFIG_PATH = options.config || node_path.join(node_os.homedir(), '.gui-agent-cli.json');
|
|
17
75
|
let config = {
|
|
18
76
|
baseURL: '',
|
|
@@ -20,7 +78,7 @@ const start = async (options)=>{
|
|
|
20
78
|
model: '',
|
|
21
79
|
provider: 'openai',
|
|
22
80
|
useResponsesApi: false,
|
|
23
|
-
maxLoopCount: 1000
|
|
81
|
+
maxLoopCount: options.maxLoopCount ? Number(options.maxLoopCount) : 1000
|
|
24
82
|
};
|
|
25
83
|
if (options.presets) {
|
|
26
84
|
const response = await node_fetch(options.presets);
|
|
@@ -272,111 +330,345 @@ const start = async (options)=>{
|
|
|
272
330
|
node_fs.mkdirSync(targetOutputDir, {
|
|
273
331
|
recursive: true
|
|
274
332
|
});
|
|
275
|
-
for (const task of tasks)
|
|
276
|
-
const
|
|
333
|
+
for (const task of tasks){
|
|
334
|
+
const taskAC = new AbortController();
|
|
335
|
+
const timeoutMs = task.timeout ?? 1500000;
|
|
336
|
+
const timeoutId = setTimeout(()=>{
|
|
337
|
+
console.log(`[CLI] Task ${task.taskId} timed out after ${timeoutMs}ms`);
|
|
338
|
+
taskAC.abort();
|
|
339
|
+
}, timeoutMs);
|
|
340
|
+
const onGlobalAbort = ()=>taskAC.abort();
|
|
341
|
+
abortController.signal.addEventListener('abort', onGlobalAbort);
|
|
342
|
+
const taskAgent = new GUIAgent({
|
|
343
|
+
model: {
|
|
344
|
+
id: config.model,
|
|
345
|
+
provider: config.provider,
|
|
346
|
+
baseURL: config.baseURL,
|
|
347
|
+
apiKey: config.apiKey
|
|
348
|
+
},
|
|
349
|
+
operator: targetOperator,
|
|
350
|
+
systemPrompt: systemPrompts.join('\n\n'),
|
|
351
|
+
signal: taskAC.signal
|
|
352
|
+
});
|
|
353
|
+
let resultEvent;
|
|
354
|
+
const startTime = Date.now();
|
|
355
|
+
try {
|
|
356
|
+
console.log(`[CLI] Starting task: ${task.taskId} at ${new Date(startTime).toISOString()}`);
|
|
357
|
+
resultEvent = await taskAgent.run(task.query);
|
|
358
|
+
} catch (taskErr) {
|
|
359
|
+
if (taskAC.signal.aborted) {
|
|
360
|
+
console.warn(`[CLI] Task ${task.taskId} was aborted (Timeout or SIGINT).`);
|
|
361
|
+
resultEvent = {
|
|
362
|
+
content: 'Task aborted or timed out'
|
|
363
|
+
};
|
|
364
|
+
} else {
|
|
365
|
+
console.error(`[CLI] Task failed: ${task.taskId}`, taskErr);
|
|
366
|
+
resultEvent = {
|
|
367
|
+
content: `Error: ${taskErr.message}`
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
} finally{
|
|
371
|
+
clearTimeout(timeoutId);
|
|
372
|
+
abortController.signal.removeEventListener('abort', onGlobalAbort);
|
|
373
|
+
}
|
|
374
|
+
try {
|
|
375
|
+
const endTime = Date.now();
|
|
376
|
+
const duration = endTime - startTime;
|
|
377
|
+
const eventStream = taskAgent.getEventStream();
|
|
378
|
+
const allEvents = eventStream.getEvents();
|
|
379
|
+
const runStartEvents = allEvents.filter((e)=>'agent_run_start' === e.type);
|
|
380
|
+
const lastRunStart = runStartEvents[runStartEvents.length - 1];
|
|
381
|
+
const startIndex = allEvents.findIndex((e)=>e.id === (null == lastRunStart ? void 0 : lastRunStart.id));
|
|
382
|
+
const endIndex = allEvents.findIndex((e, idx)=>idx > startIndex && 'agent_run_end' === e.type);
|
|
383
|
+
const rangeEvents = startIndex >= 0 ? endIndex >= 0 ? allEvents.slice(startIndex, endIndex + 1) : allEvents.slice(startIndex) : allEvents;
|
|
384
|
+
const envEvents = rangeEvents.filter((e)=>'environment_input' === e.type);
|
|
385
|
+
const screenshotEvents = envEvents.filter((e)=>e.metadata && 'screenshot' === e.metadata.type);
|
|
386
|
+
let videoPath = '';
|
|
387
|
+
if (screenshotEvents.length > 0) {
|
|
388
|
+
const tempDir = node_path.join(node_os.tmpdir(), 'gui-agent-rec', task.taskId);
|
|
389
|
+
try {
|
|
390
|
+
node_fs.mkdirSync(tempDir, {
|
|
391
|
+
recursive: true
|
|
392
|
+
});
|
|
393
|
+
const validFrames = [];
|
|
394
|
+
let frameCount = 0;
|
|
395
|
+
for (const event of screenshotEvents)if (Array.isArray(event.content)) {
|
|
396
|
+
var _imgPart_image_url;
|
|
397
|
+
const imgPart = event.content.find((c)=>'image_url' === c.type && c.image_url && c.image_url.url);
|
|
398
|
+
const dataUri = null == imgPart ? void 0 : null == (_imgPart_image_url = imgPart.image_url) ? void 0 : _imgPart_image_url.url;
|
|
399
|
+
if (dataUri && 'string' == typeof dataUri && dataUri.startsWith('data:')) {
|
|
400
|
+
const commaIndex = dataUri.indexOf(',');
|
|
401
|
+
const base64Data = commaIndex >= 0 ? dataUri.substring(commaIndex + 1) : dataUri;
|
|
402
|
+
const buffer = Buffer.from(base64Data, 'base64');
|
|
403
|
+
if (buffer.length > 0) {
|
|
404
|
+
const extension = 0xff === buffer[0] && 0xd8 === buffer[1] && 0xff === buffer[2] ? 'jpg' : 'png';
|
|
405
|
+
const fileName = `${String(frameCount).padStart(4, '0')}.${extension}`;
|
|
406
|
+
const framePath = node_path.join(tempDir, fileName);
|
|
407
|
+
node_fs.writeFileSync(framePath, buffer);
|
|
408
|
+
validFrames.push({
|
|
409
|
+
file: fileName,
|
|
410
|
+
timestamp: event.timestamp || Date.now()
|
|
411
|
+
});
|
|
412
|
+
frameCount++;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (validFrames.length > 0) {
|
|
417
|
+
const concatFilePath = node_path.join(tempDir, 'filelist.txt');
|
|
418
|
+
let fileContent = '';
|
|
419
|
+
const hasTimestamps = validFrames.some((f, i)=>i > 0 && f.timestamp !== validFrames[0].timestamp);
|
|
420
|
+
for(let i = 0; i < validFrames.length; i++){
|
|
421
|
+
const frame = validFrames[i];
|
|
422
|
+
let duration = 1.0;
|
|
423
|
+
if (hasTimestamps && i < validFrames.length - 1) {
|
|
424
|
+
const diff = (validFrames[i + 1].timestamp - frame.timestamp) / 1000;
|
|
425
|
+
if (diff > 0.1 && diff < 60) duration = diff;
|
|
426
|
+
} else if (i === validFrames.length - 1) duration = 2.0;
|
|
427
|
+
fileContent += `file '${frame.file}'\n`;
|
|
428
|
+
fileContent += `duration ${duration.toFixed(3)}\n`;
|
|
429
|
+
}
|
|
430
|
+
if (validFrames.length > 0) fileContent += `file '${validFrames[validFrames.length - 1].file}'\n`;
|
|
431
|
+
node_fs.writeFileSync(concatFilePath, fileContent);
|
|
432
|
+
const outputVideoPath = node_path.join(targetOutputDir, `${task.taskId}.mp4`);
|
|
433
|
+
console.log(`[CLI] Generating video recording: ${outputVideoPath}`);
|
|
434
|
+
await new Promise((resolve, reject)=>{
|
|
435
|
+
fluent_ffmpeg().input(concatFilePath).inputOptions([
|
|
436
|
+
'-f',
|
|
437
|
+
'concat',
|
|
438
|
+
'-safe',
|
|
439
|
+
'0'
|
|
440
|
+
]).output(outputVideoPath).outputOptions([
|
|
441
|
+
'-c:v',
|
|
442
|
+
'libx264',
|
|
443
|
+
'-pix_fmt',
|
|
444
|
+
'yuv420p',
|
|
445
|
+
'-vf',
|
|
446
|
+
'scale=trunc(iw/2)*2:trunc(ih/2)*2'
|
|
447
|
+
]).on('start', (cmd)=>console.log(`[CLI] Ffmpeg command: ${cmd}`)).on('stderr', (line)=>console.log(`[CLI] Ffmpeg stderr: ${line}`)).on('end', ()=>resolve()).on('error', (err)=>{
|
|
448
|
+
console.error('[CLI] Ffmpeg error:', err);
|
|
449
|
+
reject(err);
|
|
450
|
+
}).run();
|
|
451
|
+
});
|
|
452
|
+
videoPath = outputVideoPath;
|
|
453
|
+
console.log(`[CLI] Video saved: ${videoPath}`);
|
|
454
|
+
}
|
|
455
|
+
} catch (recErr) {
|
|
456
|
+
console.warn('[CLI] Failed to generate video recording', recErr);
|
|
457
|
+
} finally{
|
|
458
|
+
try {
|
|
459
|
+
node_fs.rmSync(tempDir, {
|
|
460
|
+
recursive: true,
|
|
461
|
+
force: true
|
|
462
|
+
});
|
|
463
|
+
} catch (_) {}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
const lastScreenshot = screenshotEvents.length > 0 ? screenshotEvents[screenshotEvents.length - 1] : null;
|
|
467
|
+
let resultPicPath = '';
|
|
468
|
+
if (lastScreenshot && Array.isArray(lastScreenshot.content)) {
|
|
469
|
+
var _imgPart_image_url1;
|
|
470
|
+
const imgPart = lastScreenshot.content.find((c)=>'image_url' === c.type && c.image_url && c.image_url.url);
|
|
471
|
+
const dataUri = null == imgPart ? void 0 : null == (_imgPart_image_url1 = imgPart.image_url) ? void 0 : _imgPart_image_url1.url;
|
|
472
|
+
if (dataUri && 'string' == typeof dataUri && dataUri.startsWith('data:')) {
|
|
473
|
+
const commaIndex = dataUri.indexOf(',');
|
|
474
|
+
const base64Data = commaIndex >= 0 ? dataUri.substring(commaIndex + 1) : dataUri;
|
|
475
|
+
const buffer = Buffer.from(base64Data, 'base64');
|
|
476
|
+
resultPicPath = node_path.join(targetOutputDir, `${task.taskId}.png`);
|
|
477
|
+
node_fs.writeFileSync(resultPicPath, buffer);
|
|
478
|
+
console.log(`[CLI] Screenshot saved: ${resultPicPath}`);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
if (!resultPicPath) console.log('[CLI] No screenshot captured; resultPic will be empty.');
|
|
482
|
+
const conversationLogPath = saveConversationLog(rangeEvents, targetOutputDir, task.taskId);
|
|
483
|
+
const finalAnswer = (null == resultEvent ? void 0 : resultEvent.content) ?? '';
|
|
484
|
+
const report = {
|
|
485
|
+
taskId: task.taskId,
|
|
486
|
+
taskContent: task.query,
|
|
487
|
+
startTime,
|
|
488
|
+
endTime,
|
|
489
|
+
duration,
|
|
490
|
+
resultPic: resultPicPath,
|
|
491
|
+
video: videoPath || null,
|
|
492
|
+
conversationLog: conversationLogPath,
|
|
493
|
+
finalAnswer
|
|
494
|
+
};
|
|
495
|
+
const reportPath = node_path.join(targetOutputDir, `${task.taskId}.json`);
|
|
496
|
+
node_fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|
497
|
+
console.log(`Result saved: ${reportPath}`);
|
|
498
|
+
console.log(`[CLI] Report JSON path: ${reportPath}`);
|
|
499
|
+
} catch (reportErr) {
|
|
500
|
+
console.warn(`[CLI] Failed to save report for task ${task.taskId}`, reportErr);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
let resultEvent;
|
|
506
|
+
const startTime = Date.now();
|
|
507
|
+
let isReportSaved = false;
|
|
508
|
+
const saveSingleTaskReport = async (finalAnswer)=>{
|
|
509
|
+
if (isReportSaved) return;
|
|
510
|
+
isReportSaved = true;
|
|
511
|
+
const endTime = Date.now();
|
|
512
|
+
const duration = endTime - startTime;
|
|
513
|
+
try {
|
|
277
514
|
const eventStream = guiAgent.getEventStream();
|
|
278
515
|
const allEvents = eventStream.getEvents();
|
|
279
516
|
const runStartEvents = allEvents.filter((e)=>'agent_run_start' === e.type);
|
|
280
|
-
const
|
|
281
|
-
const
|
|
282
|
-
const endIndex = allEvents.findIndex((e, idx)=>idx > startIndex && 'agent_run_end' === e.type);
|
|
283
|
-
const rangeEvents = startIndex >= 0 ? endIndex >= 0 ? allEvents.slice(startIndex, endIndex + 1) : allEvents.slice(startIndex) : allEvents;
|
|
284
|
-
const envEvents = rangeEvents.filter((e)=>'environment_input' === e.type);
|
|
517
|
+
const sessionId = runStartEvents.length > 0 ? runStartEvents[runStartEvents.length - 1].sessionId : `${Date.now()}`;
|
|
518
|
+
const envEvents = allEvents.filter((e)=>'environment_input' === e.type);
|
|
285
519
|
const screenshotEvents = envEvents.filter((e)=>e.metadata && 'screenshot' === e.metadata.type);
|
|
286
520
|
const lastScreenshot = screenshotEvents.length > 0 ? screenshotEvents[screenshotEvents.length - 1] : null;
|
|
521
|
+
const targetOutputDir = options.output ? node_path.resolve(options.output) : node_path.join(node_os.homedir(), '.gui-agent-results');
|
|
522
|
+
console.log(`[CLI] Output directory (resolved): ${targetOutputDir}`);
|
|
523
|
+
node_fs.mkdirSync(targetOutputDir, {
|
|
524
|
+
recursive: true
|
|
525
|
+
});
|
|
526
|
+
console.log(`[CLI] TaskId/SessionId: ${sessionId}`);
|
|
527
|
+
let videoPath = '';
|
|
528
|
+
if (screenshotEvents.length > 0) {
|
|
529
|
+
const tempDir = node_path.join(node_os.tmpdir(), 'gui-agent-rec', sessionId);
|
|
530
|
+
try {
|
|
531
|
+
node_fs.mkdirSync(tempDir, {
|
|
532
|
+
recursive: true
|
|
533
|
+
});
|
|
534
|
+
const validFrames = [];
|
|
535
|
+
let frameCount = 0;
|
|
536
|
+
for (const event of screenshotEvents)if (Array.isArray(event.content)) {
|
|
537
|
+
var _imgPart_image_url;
|
|
538
|
+
const imgPart = event.content.find((c)=>'image_url' === c.type && c.image_url && c.image_url.url);
|
|
539
|
+
const dataUri = null == imgPart ? void 0 : null == (_imgPart_image_url = imgPart.image_url) ? void 0 : _imgPart_image_url.url;
|
|
540
|
+
if (dataUri && 'string' == typeof dataUri && dataUri.startsWith('data:')) {
|
|
541
|
+
const commaIndex = dataUri.indexOf(',');
|
|
542
|
+
const base64Data = commaIndex >= 0 ? dataUri.substring(commaIndex + 1) : dataUri;
|
|
543
|
+
const buffer = Buffer.from(base64Data, 'base64');
|
|
544
|
+
if (buffer.length > 0) {
|
|
545
|
+
const extension = 0xff === buffer[0] && 0xd8 === buffer[1] && 0xff === buffer[2] ? 'jpg' : 'png';
|
|
546
|
+
const fileName = `${String(frameCount).padStart(4, '0')}.${extension}`;
|
|
547
|
+
const framePath = node_path.join(tempDir, fileName);
|
|
548
|
+
node_fs.writeFileSync(framePath, buffer);
|
|
549
|
+
validFrames.push({
|
|
550
|
+
file: fileName,
|
|
551
|
+
timestamp: event.timestamp || Date.now()
|
|
552
|
+
});
|
|
553
|
+
frameCount++;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (validFrames.length > 0) {
|
|
558
|
+
const concatFilePath = node_path.join(tempDir, 'filelist.txt');
|
|
559
|
+
let fileContent = '';
|
|
560
|
+
const hasTimestamps = validFrames.some((f, i)=>i > 0 && f.timestamp !== validFrames[0].timestamp);
|
|
561
|
+
for(let i = 0; i < validFrames.length; i++){
|
|
562
|
+
const frame = validFrames[i];
|
|
563
|
+
let duration = 1.0;
|
|
564
|
+
if (hasTimestamps && i < validFrames.length - 1) {
|
|
565
|
+
const diff = (validFrames[i + 1].timestamp - frame.timestamp) / 1000;
|
|
566
|
+
if (diff > 0.1 && diff < 60) duration = diff;
|
|
567
|
+
} else if (i === validFrames.length - 1) duration = 2.0;
|
|
568
|
+
fileContent += `file '${frame.file}'\n`;
|
|
569
|
+
fileContent += `duration ${duration.toFixed(3)}\n`;
|
|
570
|
+
}
|
|
571
|
+
if (validFrames.length > 0) fileContent += `file '${validFrames[validFrames.length - 1].file}'\n`;
|
|
572
|
+
node_fs.writeFileSync(concatFilePath, fileContent);
|
|
573
|
+
const outputVideoPath = node_path.join(targetOutputDir, `${sessionId}.mp4`);
|
|
574
|
+
console.log(`[CLI] Generating video recording: ${outputVideoPath}`);
|
|
575
|
+
await new Promise((resolve, reject)=>{
|
|
576
|
+
fluent_ffmpeg().input(concatFilePath).inputOptions([
|
|
577
|
+
'-f',
|
|
578
|
+
'concat',
|
|
579
|
+
'-safe',
|
|
580
|
+
'0'
|
|
581
|
+
]).output(outputVideoPath).outputOptions([
|
|
582
|
+
'-c:v',
|
|
583
|
+
'libx264',
|
|
584
|
+
'-pix_fmt',
|
|
585
|
+
'yuv420p',
|
|
586
|
+
'-vf',
|
|
587
|
+
'scale=trunc(iw/2)*2:trunc(ih/2)*2'
|
|
588
|
+
]).on('start', (cmd)=>console.log(`[CLI] Ffmpeg command: ${cmd}`)).on('stderr', (line)=>console.log(`[CLI] Ffmpeg stderr: ${line}`)).on('end', ()=>resolve()).on('error', (err)=>{
|
|
589
|
+
console.error('[CLI] Ffmpeg error:', err);
|
|
590
|
+
reject(err);
|
|
591
|
+
}).run();
|
|
592
|
+
});
|
|
593
|
+
videoPath = outputVideoPath;
|
|
594
|
+
console.log(`[CLI] Video saved: ${videoPath}`);
|
|
595
|
+
}
|
|
596
|
+
} catch (recErr) {
|
|
597
|
+
console.warn('[CLI] Failed to generate video recording', recErr);
|
|
598
|
+
} finally{
|
|
599
|
+
try {
|
|
600
|
+
node_fs.rmSync(tempDir, {
|
|
601
|
+
recursive: true,
|
|
602
|
+
force: true
|
|
603
|
+
});
|
|
604
|
+
} catch (_) {}
|
|
605
|
+
}
|
|
606
|
+
}
|
|
287
607
|
let resultPicPath = '';
|
|
288
608
|
if (lastScreenshot && Array.isArray(lastScreenshot.content)) {
|
|
289
|
-
var
|
|
609
|
+
var _imgPart_image_url1;
|
|
290
610
|
const imgPart = lastScreenshot.content.find((c)=>'image_url' === c.type && c.image_url && c.image_url.url);
|
|
291
|
-
const dataUri = null == imgPart ? void 0 : null == (
|
|
611
|
+
const dataUri = null == imgPart ? void 0 : null == (_imgPart_image_url1 = imgPart.image_url) ? void 0 : _imgPart_image_url1.url;
|
|
292
612
|
if (dataUri && 'string' == typeof dataUri && dataUri.startsWith('data:')) {
|
|
293
613
|
const commaIndex = dataUri.indexOf(',');
|
|
294
614
|
const base64Data = commaIndex >= 0 ? dataUri.substring(commaIndex + 1) : dataUri;
|
|
295
615
|
const buffer = Buffer.from(base64Data, 'base64');
|
|
296
|
-
resultPicPath = node_path.join(targetOutputDir, `${
|
|
616
|
+
resultPicPath = node_path.join(targetOutputDir, `${sessionId}.png`);
|
|
297
617
|
node_fs.writeFileSync(resultPicPath, buffer);
|
|
298
618
|
console.log(`[CLI] Screenshot saved: ${resultPicPath}`);
|
|
299
619
|
}
|
|
300
620
|
}
|
|
301
621
|
if (!resultPicPath) console.log('[CLI] No screenshot captured; resultPic will be empty.');
|
|
302
|
-
const
|
|
622
|
+
const conversationLogPath = saveConversationLog(allEvents, targetOutputDir, sessionId);
|
|
303
623
|
const report = {
|
|
304
|
-
taskId:
|
|
624
|
+
taskId: sessionId,
|
|
625
|
+
taskContent: answers.instruction || options.query,
|
|
626
|
+
startTime,
|
|
627
|
+
endTime,
|
|
628
|
+
duration,
|
|
305
629
|
resultPic: resultPicPath,
|
|
630
|
+
video: videoPath || null,
|
|
631
|
+
conversationLog: conversationLogPath,
|
|
306
632
|
finalAnswer
|
|
307
633
|
};
|
|
308
|
-
const reportPath = node_path.join(targetOutputDir, `${
|
|
634
|
+
const reportPath = node_path.join(targetOutputDir, `${sessionId}.json`);
|
|
309
635
|
node_fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|
310
636
|
console.log(`Result saved: ${reportPath}`);
|
|
311
637
|
console.log(`[CLI] Report JSON path: ${reportPath}`);
|
|
312
|
-
} catch (
|
|
313
|
-
console.
|
|
638
|
+
} catch (err) {
|
|
639
|
+
console.warn('Failed to generate result report:', err);
|
|
314
640
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
641
|
+
};
|
|
642
|
+
process.removeAllListeners('SIGINT');
|
|
643
|
+
process.on('SIGINT', async ()=>{
|
|
644
|
+
console.log('\n[CLI] Received SIGINT. Saving report and exiting...');
|
|
645
|
+
abortController.abort();
|
|
646
|
+
await saveSingleTaskReport("\u7528\u6237\u5DF2\u624B\u52A8\u7EC8\u6B62");
|
|
647
|
+
process.exit(0);
|
|
648
|
+
});
|
|
318
649
|
try {
|
|
319
650
|
console.log('[CLI] Starting GUIAgent run with instruction:', answers.instruction || options.query);
|
|
320
651
|
resultEvent = await guiAgent.run(answers.instruction);
|
|
321
652
|
console.log('[CLI] GUIAgent run completed.');
|
|
322
653
|
} catch (err) {
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
}
|
|
338
|
-
try {
|
|
339
|
-
const eventStream = guiAgent.getEventStream();
|
|
340
|
-
const allEvents = eventStream.getEvents();
|
|
341
|
-
const runStartEvents = allEvents.filter((e)=>'agent_run_start' === e.type);
|
|
342
|
-
const sessionId = runStartEvents.length > 0 ? runStartEvents[runStartEvents.length - 1].sessionId : `${Date.now()}`;
|
|
343
|
-
const envEvents = allEvents.filter((e)=>'environment_input' === e.type);
|
|
344
|
-
const screenshotEvents = envEvents.filter((e)=>e.metadata && 'screenshot' === e.metadata.type);
|
|
345
|
-
const lastScreenshot = screenshotEvents.length > 0 ? screenshotEvents[screenshotEvents.length - 1] : null;
|
|
346
|
-
const targetOutputDir = options.output ? node_path.resolve(options.output) : node_path.join(node_os.homedir(), '.gui-agent-results');
|
|
347
|
-
console.log(`[CLI] Output directory (resolved): ${targetOutputDir}`);
|
|
348
|
-
node_fs.mkdirSync(targetOutputDir, {
|
|
349
|
-
recursive: true
|
|
350
|
-
});
|
|
351
|
-
console.log(`[CLI] TaskId/SessionId: ${sessionId}`);
|
|
352
|
-
let resultPicPath = '';
|
|
353
|
-
if (lastScreenshot && Array.isArray(lastScreenshot.content)) {
|
|
354
|
-
var _imgPart_image_url1;
|
|
355
|
-
const imgPart = lastScreenshot.content.find((c)=>'image_url' === c.type && c.image_url && c.image_url.url);
|
|
356
|
-
const dataUri = null == imgPart ? void 0 : null == (_imgPart_image_url1 = imgPart.image_url) ? void 0 : _imgPart_image_url1.url;
|
|
357
|
-
if (dataUri && 'string' == typeof dataUri && dataUri.startsWith('data:')) {
|
|
358
|
-
const commaIndex = dataUri.indexOf(',');
|
|
359
|
-
const base64Data = commaIndex >= 0 ? dataUri.substring(commaIndex + 1) : dataUri;
|
|
360
|
-
const buffer = Buffer.from(base64Data, 'base64');
|
|
361
|
-
resultPicPath = node_path.join(targetOutputDir, `${sessionId}.png`);
|
|
362
|
-
node_fs.writeFileSync(resultPicPath, buffer);
|
|
363
|
-
console.log(`[CLI] Screenshot saved: ${resultPicPath}`);
|
|
654
|
+
if ('AbortError' === err.name || abortController.signal.aborted) console.log('[CLI] GUIAgent run aborted.');
|
|
655
|
+
else {
|
|
656
|
+
var _err_response, _err_response1;
|
|
657
|
+
console.error('[CLI] GUIAgent run failed.');
|
|
658
|
+
const errMsg = (null == err ? void 0 : err.message) || String(err);
|
|
659
|
+
console.error('[CLI] Error message:', errMsg);
|
|
660
|
+
if (null == err ? void 0 : err.status) console.error('[CLI] HTTP status:', err.status);
|
|
661
|
+
if (null == err ? void 0 : err.code) console.error('[CLI] Error code:', err.code);
|
|
662
|
+
const respData = (null == err ? void 0 : null == (_err_response = err.response) ? void 0 : _err_response.data) || (null == err ? void 0 : null == (_err_response1 = err.response) ? void 0 : _err_response1.body) || (null == err ? void 0 : err.data);
|
|
663
|
+
if (respData) try {
|
|
664
|
+
const text = 'string' == typeof respData ? respData : JSON.stringify(respData);
|
|
665
|
+
console.error('[CLI] Response body:', text.slice(0, 500));
|
|
666
|
+
} catch (_) {
|
|
667
|
+
console.error('[CLI] Response body: [unprintable]');
|
|
364
668
|
}
|
|
365
669
|
}
|
|
366
|
-
if (!resultPicPath) console.log('[CLI] No screenshot captured; resultPic will be empty.');
|
|
367
|
-
const finalAnswer = (null == resultEvent ? void 0 : resultEvent.content) ?? '';
|
|
368
|
-
const report = {
|
|
369
|
-
taskId: sessionId,
|
|
370
|
-
resultPic: resultPicPath,
|
|
371
|
-
finalAnswer
|
|
372
|
-
};
|
|
373
|
-
const reportPath = node_path.join(targetOutputDir, `${sessionId}.json`);
|
|
374
|
-
node_fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|
375
|
-
console.log(`Result saved: ${reportPath}`);
|
|
376
|
-
console.log(`[CLI] Report JSON path: ${reportPath}`);
|
|
377
|
-
} catch (err) {
|
|
378
|
-
console.warn('Failed to generate result report:', err);
|
|
379
670
|
}
|
|
671
|
+
await saveSingleTaskReport((null == resultEvent ? void 0 : resultEvent.content) ?? '');
|
|
380
672
|
};
|
|
381
673
|
const resetConfig = async (configPath)=>{
|
|
382
674
|
const CONFIG_PATH = configPath || node_path.join(node_os.homedir(), '.gui-agent-cli.json');
|