misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
@@ -45,7 +45,8 @@ var WebElementInfo = class {
|
|
45
45
|
id,
|
46
46
|
attributes,
|
47
47
|
indexId,
|
48
|
-
xpaths
|
48
|
+
xpaths,
|
49
|
+
isVisible
|
49
50
|
}) {
|
50
51
|
this.content = content;
|
51
52
|
this.rect = rect;
|
@@ -58,6 +59,7 @@ var WebElementInfo = class {
|
|
58
59
|
this.attributes = attributes;
|
59
60
|
this.indexId = indexId;
|
60
61
|
this.xpaths = xpaths;
|
62
|
+
this.isVisible = isVisible;
|
61
63
|
}
|
62
64
|
};
|
63
65
|
|
@@ -80,14 +82,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
80
82
|
})
|
81
83
|
]);
|
82
84
|
const webTree = traverseTree(tree, (elementInfo) => {
|
83
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
85
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
84
86
|
return new WebElementInfo({
|
85
87
|
rect,
|
86
88
|
locator,
|
87
89
|
id,
|
88
90
|
content,
|
89
91
|
attributes,
|
90
|
-
indexId
|
92
|
+
indexId,
|
93
|
+
isVisible
|
91
94
|
});
|
92
95
|
});
|
93
96
|
assert(screenshotBase64, "screenshotBase64 is required");
|
@@ -118,7 +121,7 @@ function printReportMsg(filepath) {
|
|
118
121
|
}
|
119
122
|
var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
|
120
123
|
function replaceIllegalPathCharsAndSpace(str) {
|
121
|
-
return str.replace(/[
|
124
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
122
125
|
}
|
123
126
|
function matchElementFromPlan(planLocateParam, tree) {
|
124
127
|
if (!planLocateParam) {
|
@@ -248,6 +251,10 @@ var ScriptPlayer = class {
|
|
248
251
|
} else if ("aiQuery" in flowItem) {
|
249
252
|
const queryTask = flowItem;
|
250
253
|
const prompt = queryTask.aiQuery;
|
254
|
+
const options = {
|
255
|
+
domIncluded: queryTask.domIncluded,
|
256
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
257
|
+
};
|
251
258
|
assert2(prompt, "missing prompt for aiQuery");
|
252
259
|
assert2(
|
253
260
|
typeof prompt === "string",
|
@@ -258,6 +265,10 @@ var ScriptPlayer = class {
|
|
258
265
|
} else if ("aiNumber" in flowItem) {
|
259
266
|
const numberTask = flowItem;
|
260
267
|
const prompt = numberTask.aiNumber;
|
268
|
+
const options = {
|
269
|
+
domIncluded: numberTask.domIncluded,
|
270
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
271
|
+
};
|
261
272
|
assert2(prompt, "missing prompt for number");
|
262
273
|
assert2(
|
263
274
|
typeof prompt === "string",
|
@@ -268,6 +279,10 @@ var ScriptPlayer = class {
|
|
268
279
|
} else if ("aiString" in flowItem) {
|
269
280
|
const stringTask = flowItem;
|
270
281
|
const prompt = stringTask.aiString;
|
282
|
+
const options = {
|
283
|
+
domIncluded: stringTask.domIncluded,
|
284
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
285
|
+
};
|
271
286
|
assert2(prompt, "missing prompt for string");
|
272
287
|
assert2(
|
273
288
|
typeof prompt === "string",
|
@@ -278,6 +293,10 @@ var ScriptPlayer = class {
|
|
278
293
|
} else if ("aiBoolean" in flowItem) {
|
279
294
|
const booleanTask = flowItem;
|
280
295
|
const prompt = booleanTask.aiBoolean;
|
296
|
+
const options = {
|
297
|
+
domIncluded: booleanTask.domIncluded,
|
298
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
299
|
+
};
|
281
300
|
assert2(prompt, "missing prompt for boolean");
|
282
301
|
assert2(
|
283
302
|
typeof prompt === "string",
|
@@ -320,6 +339,9 @@ var ScriptPlayer = class {
|
|
320
339
|
} else if ("aiTap" in flowItem) {
|
321
340
|
const tapTask = flowItem;
|
322
341
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
342
|
+
} else if ("aiRightClick" in flowItem) {
|
343
|
+
const rightClickTask = flowItem;
|
344
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
323
345
|
} else if ("aiHover" in flowItem) {
|
324
346
|
const hoverTask = flowItem;
|
325
347
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -342,6 +364,11 @@ var ScriptPlayer = class {
|
|
342
364
|
evaluateJavaScriptTask.javascript
|
343
365
|
);
|
344
366
|
this.setResult(evaluateJavaScriptTask.name, result);
|
367
|
+
} else if ("logScreenshot" in flowItem) {
|
368
|
+
const logScreenshotTask = flowItem;
|
369
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
370
|
+
content: logScreenshotTask.content || ""
|
371
|
+
});
|
345
372
|
} else {
|
346
373
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
347
374
|
}
|
@@ -615,6 +642,94 @@ var replanningCountLimit = 10;
|
|
615
642
|
var isAndroidPage = (page) => {
|
616
643
|
return page.pageType === "android";
|
617
644
|
};
|
645
|
+
var WorkflowMemory = class {
|
646
|
+
constructor(config) {
|
647
|
+
this.workflows = /* @__PURE__ */ new Map();
|
648
|
+
this.config = config;
|
649
|
+
}
|
650
|
+
/**
|
651
|
+
* İş akışı hafızasını getirir
|
652
|
+
*/
|
653
|
+
getWorkflowMemory(workflowId = "default") {
|
654
|
+
const workflow = this.workflows.get(workflowId);
|
655
|
+
return workflow?.memory || [];
|
656
|
+
}
|
657
|
+
/**
|
658
|
+
* İş akışı verilerini getirir
|
659
|
+
*/
|
660
|
+
getWorkflowData(workflowId = "default") {
|
661
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
662
|
+
}
|
663
|
+
/**
|
664
|
+
* İş akışı hafızasını kaydeder
|
665
|
+
*/
|
666
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
667
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
668
|
+
workflow.memory = [...memory];
|
669
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
670
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
671
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
672
|
+
this.workflows.set(workflowId, workflow);
|
673
|
+
this.enforceRetentionPolicy();
|
674
|
+
}
|
675
|
+
/**
|
676
|
+
* İş akışı bağlamını günceller
|
677
|
+
*/
|
678
|
+
updateWorkflowContext(context, workflowId = "default") {
|
679
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
680
|
+
workflow.context = { ...workflow.context, ...context };
|
681
|
+
if (context.currentStep) {
|
682
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
683
|
+
if (!existingStep) {
|
684
|
+
workflow.steps.push({
|
685
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
686
|
+
stepName: context.currentStep,
|
687
|
+
timestamp: context.timestamp,
|
688
|
+
status: "running",
|
689
|
+
memoryItems: []
|
690
|
+
});
|
691
|
+
}
|
692
|
+
}
|
693
|
+
this.workflows.set(workflowId, workflow);
|
694
|
+
}
|
695
|
+
/**
|
696
|
+
* İş akışını temizler
|
697
|
+
*/
|
698
|
+
clearWorkflow(workflowId = "default") {
|
699
|
+
this.workflows.delete(workflowId);
|
700
|
+
}
|
701
|
+
/**
|
702
|
+
* Tüm iş akışlarını temizler
|
703
|
+
*/
|
704
|
+
clearAll() {
|
705
|
+
this.workflows.clear();
|
706
|
+
}
|
707
|
+
createEmptyWorkflowData(workflowId) {
|
708
|
+
return {
|
709
|
+
workflowId,
|
710
|
+
steps: [],
|
711
|
+
memory: [],
|
712
|
+
context: {
|
713
|
+
pageInfo: { url: "", title: "" },
|
714
|
+
timestamp: Date.now()
|
715
|
+
},
|
716
|
+
metadata: {
|
717
|
+
totalSteps: 0,
|
718
|
+
completedSteps: 0,
|
719
|
+
failedSteps: 0,
|
720
|
+
startTime: Date.now()
|
721
|
+
}
|
722
|
+
};
|
723
|
+
}
|
724
|
+
enforceRetentionPolicy() {
|
725
|
+
const maxWorkflows = 10;
|
726
|
+
if (this.workflows.size > maxWorkflows) {
|
727
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
728
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
729
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
730
|
+
}
|
731
|
+
}
|
732
|
+
};
|
618
733
|
var PageTaskExecutor = class {
|
619
734
|
constructor(page, insight, opts) {
|
620
735
|
this.conversationHistory = [];
|
@@ -622,6 +737,25 @@ var PageTaskExecutor = class {
|
|
622
737
|
this.insight = insight;
|
623
738
|
this.taskCache = opts.taskCache;
|
624
739
|
this.onTaskStartCallback = opts?.onTaskStart;
|
740
|
+
this.memoryConfig = {
|
741
|
+
maxItems: 100,
|
742
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
743
|
+
// 2 saat
|
744
|
+
enablePersistence: true,
|
745
|
+
enableAnalytics: true,
|
746
|
+
filterStrategy: "hybrid",
|
747
|
+
...opts?.memoryConfig
|
748
|
+
};
|
749
|
+
this.sessionContext = {
|
750
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
751
|
+
workflowId: opts?.workflowId,
|
752
|
+
startTime: Date.now(),
|
753
|
+
pageInfo: {
|
754
|
+
url: "",
|
755
|
+
title: ""
|
756
|
+
}
|
757
|
+
};
|
758
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
625
759
|
}
|
626
760
|
async recordScreenshot(timing) {
|
627
761
|
const base64 = await this.page.screenshotBase64();
|
@@ -834,8 +968,11 @@ var PageTaskExecutor = class {
|
|
834
968
|
insightDump = dump;
|
835
969
|
};
|
836
970
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
971
|
+
const memoryContext = this.getMemoryAsContext();
|
837
972
|
const assertion = await this.insight.assert(
|
838
|
-
assertPlan.param.assertion
|
973
|
+
assertPlan.param.assertion,
|
974
|
+
memoryContext
|
975
|
+
// Hafıza bağlamını geç
|
839
976
|
);
|
840
977
|
if (!assertion.pass) {
|
841
978
|
if (plan2.type === "Assert") {
|
@@ -872,10 +1009,10 @@ var PageTaskExecutor = class {
|
|
872
1009
|
if (!taskParam || !taskParam.value) {
|
873
1010
|
return;
|
874
1011
|
}
|
875
|
-
await this.page.keyboard.type(taskParam.value);
|
876
|
-
} else {
|
877
|
-
await this.page.keyboard.type(taskParam.value);
|
878
1012
|
}
|
1013
|
+
await this.page.keyboard.type(taskParam.value, {
|
1014
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
1015
|
+
});
|
879
1016
|
}
|
880
1017
|
};
|
881
1018
|
tasks.push(taskActionInput);
|
@@ -904,6 +1041,22 @@ var PageTaskExecutor = class {
|
|
904
1041
|
}
|
905
1042
|
};
|
906
1043
|
tasks.push(taskActionTap);
|
1044
|
+
} else if (plan2.type === "RightClick") {
|
1045
|
+
const taskActionRightClick = {
|
1046
|
+
type: "Action",
|
1047
|
+
subType: "RightClick",
|
1048
|
+
thought: plan2.thought,
|
1049
|
+
locate: plan2.locate,
|
1050
|
+
executor: async (param, { element }) => {
|
1051
|
+
assert4(element, "Element not found, cannot right click");
|
1052
|
+
await this.page.mouse.click(
|
1053
|
+
element.center[0],
|
1054
|
+
element.center[1],
|
1055
|
+
{ button: "right" }
|
1056
|
+
);
|
1057
|
+
}
|
1058
|
+
};
|
1059
|
+
tasks.push(taskActionRightClick);
|
907
1060
|
} else if (plan2.type === "Drag") {
|
908
1061
|
const taskActionDrag = {
|
909
1062
|
type: "Action",
|
@@ -1305,25 +1458,146 @@ var PageTaskExecutor = class {
|
|
1305
1458
|
};
|
1306
1459
|
return task;
|
1307
1460
|
}
|
1461
|
+
/**
|
1462
|
+
* Persistent executor'ı getirir veya oluşturur
|
1463
|
+
*/
|
1464
|
+
getPersistentExecutor() {
|
1465
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1466
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1467
|
+
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1468
|
+
onTaskStart: this.onTaskStartCallback,
|
1469
|
+
initialMemory: previousMemory
|
1470
|
+
});
|
1471
|
+
}
|
1472
|
+
return this.persistentExecutor;
|
1473
|
+
}
|
1474
|
+
/**
|
1475
|
+
* Sayfa bağlamını günceller
|
1476
|
+
*/
|
1477
|
+
async updatePageContext() {
|
1478
|
+
try {
|
1479
|
+
if (this.page.url) {
|
1480
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1481
|
+
}
|
1482
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1483
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1484
|
+
}
|
1485
|
+
} catch (e) {
|
1486
|
+
}
|
1487
|
+
}
|
1488
|
+
/**
|
1489
|
+
* Hafızayı temizler
|
1490
|
+
*/
|
1491
|
+
clearMemory() {
|
1492
|
+
if (this.persistentExecutor) {
|
1493
|
+
this.persistentExecutor.clearMemory();
|
1494
|
+
}
|
1495
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1496
|
+
}
|
1497
|
+
/**
|
1498
|
+
* Mevcut hafızayı döndürür
|
1499
|
+
*/
|
1500
|
+
getMemory() {
|
1501
|
+
return this.persistentExecutor?.getMemory() || [];
|
1502
|
+
}
|
1503
|
+
/**
|
1504
|
+
* İş akışı hafızasını döndürür
|
1505
|
+
*/
|
1506
|
+
getWorkflowMemory() {
|
1507
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1508
|
+
}
|
1509
|
+
/**
|
1510
|
+
* Hafıza istatistiklerini döndürür
|
1511
|
+
*/
|
1512
|
+
getMemoryStats() {
|
1513
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1514
|
+
totalItems: 0,
|
1515
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1516
|
+
config: this.memoryConfig
|
1517
|
+
};
|
1518
|
+
}
|
1519
|
+
/**
|
1520
|
+
* Hafıza konfigürasyonunu günceller
|
1521
|
+
*/
|
1522
|
+
updateMemoryConfig(config) {
|
1523
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1524
|
+
if (this.persistentExecutor) {
|
1525
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1526
|
+
}
|
1527
|
+
}
|
1528
|
+
/**
|
1529
|
+
* Oturum ID'sini oluşturur
|
1530
|
+
*/
|
1531
|
+
generateSessionId() {
|
1532
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1533
|
+
}
|
1534
|
+
/**
|
1535
|
+
* Hafızayı bağlam olarak formatlar
|
1536
|
+
*/
|
1537
|
+
getMemoryAsContext() {
|
1538
|
+
const memory = this.getMemory();
|
1539
|
+
if (memory.length === 0) {
|
1540
|
+
return "";
|
1541
|
+
}
|
1542
|
+
const recentMemory = memory.slice(-5);
|
1543
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1544
|
+
}
|
1308
1545
|
async runPlans(title, plans, opts) {
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1546
|
+
await this.updatePageContext();
|
1547
|
+
const useMemory = opts?.useMemory !== false;
|
1548
|
+
let taskExecutor;
|
1549
|
+
if (useMemory) {
|
1550
|
+
taskExecutor = this.getPersistentExecutor();
|
1551
|
+
this.workflowMemory.updateWorkflowContext({
|
1552
|
+
currentStep: title,
|
1553
|
+
pageInfo: this.sessionContext.pageInfo,
|
1554
|
+
timestamp: Date.now()
|
1555
|
+
}, this.sessionContext.workflowId || "default");
|
1556
|
+
} else {
|
1557
|
+
taskExecutor = new Executor(title, {
|
1558
|
+
onTaskStart: this.onTaskStartCallback
|
1559
|
+
});
|
1560
|
+
}
|
1312
1561
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1562
|
+
tasks.forEach((task) => {
|
1563
|
+
task.context = {
|
1564
|
+
...task.context,
|
1565
|
+
...this.sessionContext.pageInfo,
|
1566
|
+
workflowId: this.sessionContext.workflowId,
|
1567
|
+
sessionId: this.sessionContext.sessionId
|
1568
|
+
};
|
1569
|
+
});
|
1313
1570
|
await taskExecutor.append(tasks);
|
1314
1571
|
const result = await taskExecutor.flush();
|
1572
|
+
if (useMemory) {
|
1573
|
+
this.workflowMemory.saveWorkflowMemory(
|
1574
|
+
taskExecutor.getMemory(),
|
1575
|
+
this.sessionContext.workflowId || "default"
|
1576
|
+
);
|
1577
|
+
}
|
1315
1578
|
return {
|
1316
1579
|
output: result,
|
1317
1580
|
executor: taskExecutor
|
1318
1581
|
};
|
1319
1582
|
}
|
1320
1583
|
async action(userPrompt, actionContext, opts) {
|
1321
|
-
const
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1584
|
+
const useMemory = true;
|
1585
|
+
let taskExecutor;
|
1586
|
+
if (useMemory) {
|
1587
|
+
taskExecutor = this.getPersistentExecutor();
|
1588
|
+
} else {
|
1589
|
+
taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
|
1590
|
+
onTaskStart: this.onTaskStartCallback
|
1591
|
+
});
|
1592
|
+
}
|
1593
|
+
const memoryContext = this.getMemoryAsContext();
|
1594
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1595
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1325
1596
|
let replanCount = 0;
|
1326
1597
|
const logList = [];
|
1598
|
+
if (memoryContext) {
|
1599
|
+
logList.push(memoryContext);
|
1600
|
+
}
|
1327
1601
|
const yamlFlow = [];
|
1328
1602
|
while (planningTask) {
|
1329
1603
|
if (replanCount > replanningCountLimit) {
|
@@ -1432,16 +1706,22 @@ var PageTaskExecutor = class {
|
|
1432
1706
|
executor: taskExecutor
|
1433
1707
|
};
|
1434
1708
|
}
|
1435
|
-
async createTypeQueryTask(type, demand) {
|
1436
|
-
const
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1709
|
+
async createTypeQueryTask(type, demand, opt) {
|
1710
|
+
const useMemory = true;
|
1711
|
+
let taskExecutor;
|
1712
|
+
if (useMemory) {
|
1713
|
+
taskExecutor = this.getPersistentExecutor();
|
1714
|
+
} else {
|
1715
|
+
taskExecutor = new Executor(
|
1716
|
+
taskTitleStr(
|
1717
|
+
type,
|
1718
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1719
|
+
),
|
1720
|
+
{
|
1721
|
+
onTaskStart: this.onTaskStartCallback
|
1722
|
+
}
|
1723
|
+
);
|
1724
|
+
}
|
1445
1725
|
const queryTask = {
|
1446
1726
|
type: "Insight",
|
1447
1727
|
subType: type,
|
@@ -1463,7 +1743,13 @@ var PageTaskExecutor = class {
|
|
1463
1743
|
result: `${type}, ${demand}`
|
1464
1744
|
};
|
1465
1745
|
}
|
1466
|
-
const
|
1746
|
+
const memoryContext = this.getMemoryAsContext();
|
1747
|
+
const { data, usage } = await this.insight.extract(
|
1748
|
+
demandInput,
|
1749
|
+
opt,
|
1750
|
+
memoryContext
|
1751
|
+
// Hafıza bağlamını geç
|
1752
|
+
);
|
1467
1753
|
let outputResult = data;
|
1468
1754
|
if (ifTypeRestricted) {
|
1469
1755
|
assert4(data?.result !== void 0, "No result in query data");
|
@@ -1483,23 +1769,29 @@ var PageTaskExecutor = class {
|
|
1483
1769
|
executor: taskExecutor
|
1484
1770
|
};
|
1485
1771
|
}
|
1486
|
-
async query(demand) {
|
1487
|
-
return this.createTypeQueryTask("Query", demand);
|
1772
|
+
async query(demand, opt) {
|
1773
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1488
1774
|
}
|
1489
|
-
async boolean(prompt) {
|
1490
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1775
|
+
async boolean(prompt, opt) {
|
1776
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1491
1777
|
}
|
1492
|
-
async number(prompt) {
|
1493
|
-
return this.createTypeQueryTask("Number", prompt);
|
1778
|
+
async number(prompt, opt) {
|
1779
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1494
1780
|
}
|
1495
|
-
async string(prompt) {
|
1496
|
-
return this.createTypeQueryTask("String", prompt);
|
1781
|
+
async string(prompt, opt) {
|
1782
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1497
1783
|
}
|
1498
|
-
async assert(assertion) {
|
1784
|
+
async assert(assertion, memoryContext) {
|
1499
1785
|
const description = `assert: ${assertion}`;
|
1500
|
-
const
|
1501
|
-
|
1502
|
-
|
1786
|
+
const useMemory = true;
|
1787
|
+
let taskExecutor;
|
1788
|
+
if (useMemory) {
|
1789
|
+
taskExecutor = this.getPersistentExecutor();
|
1790
|
+
} else {
|
1791
|
+
taskExecutor = new Executor(taskTitleStr("Assert", description), {
|
1792
|
+
onTaskStart: this.onTaskStartCallback
|
1793
|
+
});
|
1794
|
+
}
|
1503
1795
|
const assertionPlan = {
|
1504
1796
|
type: "Assert",
|
1505
1797
|
param: {
|
@@ -1629,7 +1921,7 @@ function buildPlans(type, locateParam, param) {
|
|
1629
1921
|
param: locateParam,
|
1630
1922
|
thought: ""
|
1631
1923
|
} : null;
|
1632
|
-
if (type === "Tap" || type === "Hover") {
|
1924
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1633
1925
|
assert5(locateParam, `missing locate info for action "${type}"`);
|
1634
1926
|
assert5(locatePlan, `missing locate info for action "${type}"`);
|
1635
1927
|
const tapPlan = {
|
@@ -1700,8 +1992,8 @@ function buildPlans(type, locateParam, param) {
|
|
1700
1992
|
|
1701
1993
|
// src/common/task-cache.ts
|
1702
1994
|
import assert6 from "assert";
|
1703
|
-
import { existsSync as existsSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
1704
|
-
import { join as join2 } from "path";
|
1995
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
1996
|
+
import { dirname as dirname2, join as join2 } from "path";
|
1705
1997
|
import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
|
1706
1998
|
import { getDebug as getDebug3 } from "misoai-shared/logger";
|
1707
1999
|
import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
|
@@ -1709,7 +2001,7 @@ import yaml3 from "js-yaml";
|
|
1709
2001
|
import semver from "semver";
|
1710
2002
|
|
1711
2003
|
// package.json
|
1712
|
-
var version = "1.5.
|
2004
|
+
var version = "1.5.7";
|
1713
2005
|
|
1714
2006
|
// src/common/task-cache.ts
|
1715
2007
|
var debug3 = getDebug3("cache");
|
@@ -1840,8 +2132,14 @@ cache file: ${cacheFile}`
|
|
1840
2132
|
return;
|
1841
2133
|
}
|
1842
2134
|
try {
|
2135
|
+
const dir = dirname2(this.cacheFilePath);
|
2136
|
+
if (!existsSync2(dir)) {
|
2137
|
+
mkdirSync2(dir, { recursive: true });
|
2138
|
+
debug3("created cache directory: %s", dir);
|
2139
|
+
}
|
1843
2140
|
const yamlData = yaml3.dump(this.cache);
|
1844
2141
|
writeFileSync2(this.cacheFilePath, yamlData);
|
2142
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1845
2143
|
} catch (err) {
|
1846
2144
|
debug3(
|
1847
2145
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2099,6 +2397,23 @@ var PageAgent = class {
|
|
2099
2397
|
metadata
|
2100
2398
|
};
|
2101
2399
|
}
|
2400
|
+
async aiRightClick(locatePrompt, opt) {
|
2401
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2402
|
+
locatePrompt,
|
2403
|
+
opt
|
2404
|
+
);
|
2405
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2406
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2407
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2408
|
+
plans,
|
2409
|
+
{ cacheable: opt?.cacheable }
|
2410
|
+
);
|
2411
|
+
const metadata = this.afterTaskRunning(executor);
|
2412
|
+
return {
|
2413
|
+
result: output,
|
2414
|
+
metadata
|
2415
|
+
};
|
2416
|
+
}
|
2102
2417
|
async aiInput(value, locatePrompt, opt) {
|
2103
2418
|
assert7(
|
2104
2419
|
typeof value === "string",
|
@@ -2173,7 +2488,13 @@ var PageAgent = class {
|
|
2173
2488
|
metadata: metadata2
|
2174
2489
|
};
|
2175
2490
|
}
|
2176
|
-
const
|
2491
|
+
const memoryContext = this.getMemoryAsContext();
|
2492
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2493
|
+
|
2494
|
+
Previous workflow steps:
|
2495
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2496
|
+
${memoryContext}` : void 0;
|
2497
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2177
2498
|
cacheable
|
2178
2499
|
}));
|
2179
2500
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2319,8 +2640,9 @@ var PageAgent = class {
|
|
2319
2640
|
} catch (e) {
|
2320
2641
|
}
|
2321
2642
|
}
|
2643
|
+
const memoryContext = this.getMemoryAsContext();
|
2322
2644
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2323
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2645
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2324
2646
|
const metadata = this.afterTaskRunning(executor, true);
|
2325
2647
|
if (output && opt?.keepRawResponse) {
|
2326
2648
|
return {
|
@@ -2530,9 +2852,72 @@ ${errors}`);
|
|
2530
2852
|
}
|
2531
2853
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2532
2854
|
}
|
2855
|
+
async logScreenshot(title, options) {
|
2856
|
+
const screenshotTitle = title || "untitled";
|
2857
|
+
const content = options?.content || "";
|
2858
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2859
|
+
if (screenshot) {
|
2860
|
+
const executionDump = {
|
2861
|
+
name: screenshotTitle,
|
2862
|
+
description: content,
|
2863
|
+
tasks: [{
|
2864
|
+
type: "Screenshot",
|
2865
|
+
subType: "log",
|
2866
|
+
status: "finished",
|
2867
|
+
executor: null,
|
2868
|
+
param: {
|
2869
|
+
title: screenshotTitle,
|
2870
|
+
content
|
2871
|
+
},
|
2872
|
+
output: {
|
2873
|
+
screenshot
|
2874
|
+
},
|
2875
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2876
|
+
timing: {
|
2877
|
+
start: Date.now(),
|
2878
|
+
end: Date.now(),
|
2879
|
+
cost: 0
|
2880
|
+
}
|
2881
|
+
}],
|
2882
|
+
sdkVersion: "1.0.0",
|
2883
|
+
logTime: Date.now(),
|
2884
|
+
model_name: "screenshot"
|
2885
|
+
};
|
2886
|
+
this.appendExecutionDump(executionDump);
|
2887
|
+
}
|
2888
|
+
}
|
2533
2889
|
async destroy() {
|
2534
2890
|
await this.page.destroy();
|
2535
2891
|
}
|
2892
|
+
/**
|
2893
|
+
* Hafızayı bağlam olarak formatlar
|
2894
|
+
*/
|
2895
|
+
getMemoryAsContext() {
|
2896
|
+
const memory = this.taskExecutor.getMemory();
|
2897
|
+
if (memory.length === 0) {
|
2898
|
+
return "";
|
2899
|
+
}
|
2900
|
+
const recentMemory = memory.slice(-5);
|
2901
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2902
|
+
}
|
2903
|
+
/**
|
2904
|
+
* Mevcut hafızayı döndürür
|
2905
|
+
*/
|
2906
|
+
getMemory() {
|
2907
|
+
return this.taskExecutor.getMemory();
|
2908
|
+
}
|
2909
|
+
/**
|
2910
|
+
* Hafıza istatistiklerini döndürür
|
2911
|
+
*/
|
2912
|
+
getMemoryStats() {
|
2913
|
+
return this.taskExecutor.getMemoryStats();
|
2914
|
+
}
|
2915
|
+
/**
|
2916
|
+
* Hafızayı temizler
|
2917
|
+
*/
|
2918
|
+
clearMemory() {
|
2919
|
+
this.taskExecutor.clearMemory();
|
2920
|
+
}
|
2536
2921
|
};
|
2537
2922
|
|
2538
2923
|
// src/chrome-extension/agent.ts
|
@@ -2752,7 +3137,7 @@ function sleep2(ms) {
|
|
2752
3137
|
var ChromeExtensionProxyPage = class {
|
2753
3138
|
constructor(forceSameTabNavigation) {
|
2754
3139
|
this.pageType = "chrome-extension-proxy";
|
2755
|
-
this.version = "1.5.
|
3140
|
+
this.version = "1.5.7";
|
2756
3141
|
this.activeTabId = null;
|
2757
3142
|
this.tabIdOfDebuggerAttached = null;
|
2758
3143
|
this.attachingDebugger = null;
|
@@ -2761,7 +3146,8 @@ var ChromeExtensionProxyPage = class {
|
|
2761
3146
|
this.latestMouseX = 100;
|
2762
3147
|
this.latestMouseY = 100;
|
2763
3148
|
this.mouse = {
|
2764
|
-
click: async (x, y) => {
|
3149
|
+
click: async (x, y, options) => {
|
3150
|
+
const { button = "left", count = 1 } = options || {};
|
2765
3151
|
await this.mouse.move(x, y);
|
2766
3152
|
if (this.isMobileEmulation === null) {
|
2767
3153
|
const result = await this.sendCommandToDebugger("Runtime.evaluate", {
|
@@ -2772,7 +3158,7 @@ var ChromeExtensionProxyPage = class {
|
|
2772
3158
|
});
|
2773
3159
|
this.isMobileEmulation = result?.result?.value;
|
2774
3160
|
}
|
2775
|
-
if (this.isMobileEmulation) {
|
3161
|
+
if (this.isMobileEmulation && button === "left") {
|
2776
3162
|
const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
|
2777
3163
|
await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
|
2778
3164
|
type: "touchStart",
|
@@ -2789,15 +3175,15 @@ var ChromeExtensionProxyPage = class {
|
|
2789
3175
|
type: "mousePressed",
|
2790
3176
|
x,
|
2791
3177
|
y,
|
2792
|
-
button
|
2793
|
-
clickCount:
|
3178
|
+
button,
|
3179
|
+
clickCount: count
|
2794
3180
|
});
|
2795
3181
|
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
2796
3182
|
type: "mouseReleased",
|
2797
3183
|
x,
|
2798
3184
|
y,
|
2799
|
-
button
|
2800
|
-
clickCount:
|
3185
|
+
button,
|
3186
|
+
clickCount: count
|
2801
3187
|
});
|
2802
3188
|
}
|
2803
3189
|
},
|