misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
package/dist/lib/index.js
CHANGED
@@ -148,6 +148,10 @@ var ScriptPlayer = class {
|
|
148
148
|
} else if ("aiQuery" in flowItem) {
|
149
149
|
const queryTask = flowItem;
|
150
150
|
const prompt = queryTask.aiQuery;
|
151
|
+
const options = {
|
152
|
+
domIncluded: queryTask.domIncluded,
|
153
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
154
|
+
};
|
151
155
|
(0, import_utils.assert)(prompt, "missing prompt for aiQuery");
|
152
156
|
(0, import_utils.assert)(
|
153
157
|
typeof prompt === "string",
|
@@ -158,6 +162,10 @@ var ScriptPlayer = class {
|
|
158
162
|
} else if ("aiNumber" in flowItem) {
|
159
163
|
const numberTask = flowItem;
|
160
164
|
const prompt = numberTask.aiNumber;
|
165
|
+
const options = {
|
166
|
+
domIncluded: numberTask.domIncluded,
|
167
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
168
|
+
};
|
161
169
|
(0, import_utils.assert)(prompt, "missing prompt for number");
|
162
170
|
(0, import_utils.assert)(
|
163
171
|
typeof prompt === "string",
|
@@ -168,6 +176,10 @@ var ScriptPlayer = class {
|
|
168
176
|
} else if ("aiString" in flowItem) {
|
169
177
|
const stringTask = flowItem;
|
170
178
|
const prompt = stringTask.aiString;
|
179
|
+
const options = {
|
180
|
+
domIncluded: stringTask.domIncluded,
|
181
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
182
|
+
};
|
171
183
|
(0, import_utils.assert)(prompt, "missing prompt for string");
|
172
184
|
(0, import_utils.assert)(
|
173
185
|
typeof prompt === "string",
|
@@ -178,6 +190,10 @@ var ScriptPlayer = class {
|
|
178
190
|
} else if ("aiBoolean" in flowItem) {
|
179
191
|
const booleanTask = flowItem;
|
180
192
|
const prompt = booleanTask.aiBoolean;
|
193
|
+
const options = {
|
194
|
+
domIncluded: booleanTask.domIncluded,
|
195
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
196
|
+
};
|
181
197
|
(0, import_utils.assert)(prompt, "missing prompt for boolean");
|
182
198
|
(0, import_utils.assert)(
|
183
199
|
typeof prompt === "string",
|
@@ -220,6 +236,9 @@ var ScriptPlayer = class {
|
|
220
236
|
} else if ("aiTap" in flowItem) {
|
221
237
|
const tapTask = flowItem;
|
222
238
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
239
|
+
} else if ("aiRightClick" in flowItem) {
|
240
|
+
const rightClickTask = flowItem;
|
241
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
223
242
|
} else if ("aiHover" in flowItem) {
|
224
243
|
const hoverTask = flowItem;
|
225
244
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -242,6 +261,11 @@ var ScriptPlayer = class {
|
|
242
261
|
evaluateJavaScriptTask.javascript
|
243
262
|
);
|
244
263
|
this.setResult(evaluateJavaScriptTask.name, result);
|
264
|
+
} else if ("logScreenshot" in flowItem) {
|
265
|
+
const logScreenshotTask = flowItem;
|
266
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
267
|
+
content: logScreenshotTask.content || ""
|
268
|
+
});
|
245
269
|
} else {
|
246
270
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
247
271
|
}
|
@@ -490,7 +514,8 @@ var WebElementInfo = class {
|
|
490
514
|
id,
|
491
515
|
attributes,
|
492
516
|
indexId,
|
493
|
-
xpaths
|
517
|
+
xpaths,
|
518
|
+
isVisible
|
494
519
|
}) {
|
495
520
|
this.content = content;
|
496
521
|
this.rect = rect;
|
@@ -503,6 +528,7 @@ var WebElementInfo = class {
|
|
503
528
|
this.attributes = attributes;
|
504
529
|
this.indexId = indexId;
|
505
530
|
this.xpaths = xpaths;
|
531
|
+
this.isVisible = isVisible;
|
506
532
|
}
|
507
533
|
};
|
508
534
|
|
@@ -525,14 +551,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
525
551
|
})
|
526
552
|
]);
|
527
553
|
const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
|
528
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
554
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
529
555
|
return new WebElementInfo({
|
530
556
|
rect,
|
531
557
|
locator,
|
532
558
|
id,
|
533
559
|
content,
|
534
560
|
attributes,
|
535
|
-
indexId
|
561
|
+
indexId,
|
562
|
+
isVisible
|
536
563
|
});
|
537
564
|
});
|
538
565
|
(0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
|
@@ -562,7 +589,7 @@ function printReportMsg(filepath) {
|
|
562
589
|
(0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
|
563
590
|
}
|
564
591
|
function replaceIllegalPathCharsAndSpace(str) {
|
565
|
-
return str.replace(/[
|
592
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
566
593
|
}
|
567
594
|
function forceClosePopup(page, debug7) {
|
568
595
|
page.on("popup", async (popup) => {
|
@@ -619,6 +646,94 @@ var replanningCountLimit = 10;
|
|
619
646
|
var isAndroidPage = (page) => {
|
620
647
|
return page.pageType === "android";
|
621
648
|
};
|
649
|
+
var WorkflowMemory = class {
|
650
|
+
constructor(config) {
|
651
|
+
this.workflows = /* @__PURE__ */ new Map();
|
652
|
+
this.config = config;
|
653
|
+
}
|
654
|
+
/**
|
655
|
+
* İş akışı hafızasını getirir
|
656
|
+
*/
|
657
|
+
getWorkflowMemory(workflowId = "default") {
|
658
|
+
const workflow = this.workflows.get(workflowId);
|
659
|
+
return workflow?.memory || [];
|
660
|
+
}
|
661
|
+
/**
|
662
|
+
* İş akışı verilerini getirir
|
663
|
+
*/
|
664
|
+
getWorkflowData(workflowId = "default") {
|
665
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
666
|
+
}
|
667
|
+
/**
|
668
|
+
* İş akışı hafızasını kaydeder
|
669
|
+
*/
|
670
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
671
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
672
|
+
workflow.memory = [...memory];
|
673
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
674
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
675
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
676
|
+
this.workflows.set(workflowId, workflow);
|
677
|
+
this.enforceRetentionPolicy();
|
678
|
+
}
|
679
|
+
/**
|
680
|
+
* İş akışı bağlamını günceller
|
681
|
+
*/
|
682
|
+
updateWorkflowContext(context, workflowId = "default") {
|
683
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
684
|
+
workflow.context = { ...workflow.context, ...context };
|
685
|
+
if (context.currentStep) {
|
686
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
687
|
+
if (!existingStep) {
|
688
|
+
workflow.steps.push({
|
689
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
690
|
+
stepName: context.currentStep,
|
691
|
+
timestamp: context.timestamp,
|
692
|
+
status: "running",
|
693
|
+
memoryItems: []
|
694
|
+
});
|
695
|
+
}
|
696
|
+
}
|
697
|
+
this.workflows.set(workflowId, workflow);
|
698
|
+
}
|
699
|
+
/**
|
700
|
+
* İş akışını temizler
|
701
|
+
*/
|
702
|
+
clearWorkflow(workflowId = "default") {
|
703
|
+
this.workflows.delete(workflowId);
|
704
|
+
}
|
705
|
+
/**
|
706
|
+
* Tüm iş akışlarını temizler
|
707
|
+
*/
|
708
|
+
clearAll() {
|
709
|
+
this.workflows.clear();
|
710
|
+
}
|
711
|
+
createEmptyWorkflowData(workflowId) {
|
712
|
+
return {
|
713
|
+
workflowId,
|
714
|
+
steps: [],
|
715
|
+
memory: [],
|
716
|
+
context: {
|
717
|
+
pageInfo: { url: "", title: "" },
|
718
|
+
timestamp: Date.now()
|
719
|
+
},
|
720
|
+
metadata: {
|
721
|
+
totalSteps: 0,
|
722
|
+
completedSteps: 0,
|
723
|
+
failedSteps: 0,
|
724
|
+
startTime: Date.now()
|
725
|
+
}
|
726
|
+
};
|
727
|
+
}
|
728
|
+
enforceRetentionPolicy() {
|
729
|
+
const maxWorkflows = 10;
|
730
|
+
if (this.workflows.size > maxWorkflows) {
|
731
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
732
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
733
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
734
|
+
}
|
735
|
+
}
|
736
|
+
};
|
622
737
|
var PageTaskExecutor = class {
|
623
738
|
constructor(page, insight, opts) {
|
624
739
|
this.conversationHistory = [];
|
@@ -626,6 +741,25 @@ var PageTaskExecutor = class {
|
|
626
741
|
this.insight = insight;
|
627
742
|
this.taskCache = opts.taskCache;
|
628
743
|
this.onTaskStartCallback = opts?.onTaskStart;
|
744
|
+
this.memoryConfig = {
|
745
|
+
maxItems: 100,
|
746
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
747
|
+
// 2 saat
|
748
|
+
enablePersistence: true,
|
749
|
+
enableAnalytics: true,
|
750
|
+
filterStrategy: "hybrid",
|
751
|
+
...opts?.memoryConfig
|
752
|
+
};
|
753
|
+
this.sessionContext = {
|
754
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
755
|
+
workflowId: opts?.workflowId,
|
756
|
+
startTime: Date.now(),
|
757
|
+
pageInfo: {
|
758
|
+
url: "",
|
759
|
+
title: ""
|
760
|
+
}
|
761
|
+
};
|
762
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
629
763
|
}
|
630
764
|
async recordScreenshot(timing) {
|
631
765
|
const base64 = await this.page.screenshotBase64();
|
@@ -838,8 +972,11 @@ var PageTaskExecutor = class {
|
|
838
972
|
insightDump = dump;
|
839
973
|
};
|
840
974
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
975
|
+
const memoryContext = this.getMemoryAsContext();
|
841
976
|
const assertion = await this.insight.assert(
|
842
|
-
assertPlan.param.assertion
|
977
|
+
assertPlan.param.assertion,
|
978
|
+
memoryContext
|
979
|
+
// Hafıza bağlamını geç
|
843
980
|
);
|
844
981
|
if (!assertion.pass) {
|
845
982
|
if (plan2.type === "Assert") {
|
@@ -876,10 +1013,10 @@ var PageTaskExecutor = class {
|
|
876
1013
|
if (!taskParam || !taskParam.value) {
|
877
1014
|
return;
|
878
1015
|
}
|
879
|
-
await this.page.keyboard.type(taskParam.value);
|
880
|
-
} else {
|
881
|
-
await this.page.keyboard.type(taskParam.value);
|
882
1016
|
}
|
1017
|
+
await this.page.keyboard.type(taskParam.value, {
|
1018
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
1019
|
+
});
|
883
1020
|
}
|
884
1021
|
};
|
885
1022
|
tasks.push(taskActionInput);
|
@@ -908,6 +1045,22 @@ var PageTaskExecutor = class {
|
|
908
1045
|
}
|
909
1046
|
};
|
910
1047
|
tasks.push(taskActionTap);
|
1048
|
+
} else if (plan2.type === "RightClick") {
|
1049
|
+
const taskActionRightClick = {
|
1050
|
+
type: "Action",
|
1051
|
+
subType: "RightClick",
|
1052
|
+
thought: plan2.thought,
|
1053
|
+
locate: plan2.locate,
|
1054
|
+
executor: async (param, { element }) => {
|
1055
|
+
(0, import_utils6.assert)(element, "Element not found, cannot right click");
|
1056
|
+
await this.page.mouse.click(
|
1057
|
+
element.center[0],
|
1058
|
+
element.center[1],
|
1059
|
+
{ button: "right" }
|
1060
|
+
);
|
1061
|
+
}
|
1062
|
+
};
|
1063
|
+
tasks.push(taskActionRightClick);
|
911
1064
|
} else if (plan2.type === "Drag") {
|
912
1065
|
const taskActionDrag = {
|
913
1066
|
type: "Action",
|
@@ -1309,25 +1462,146 @@ var PageTaskExecutor = class {
|
|
1309
1462
|
};
|
1310
1463
|
return task;
|
1311
1464
|
}
|
1465
|
+
/**
|
1466
|
+
* Persistent executor'ı getirir veya oluşturur
|
1467
|
+
*/
|
1468
|
+
getPersistentExecutor() {
|
1469
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1470
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1471
|
+
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1472
|
+
onTaskStart: this.onTaskStartCallback,
|
1473
|
+
initialMemory: previousMemory
|
1474
|
+
});
|
1475
|
+
}
|
1476
|
+
return this.persistentExecutor;
|
1477
|
+
}
|
1478
|
+
/**
|
1479
|
+
* Sayfa bağlamını günceller
|
1480
|
+
*/
|
1481
|
+
async updatePageContext() {
|
1482
|
+
try {
|
1483
|
+
if (this.page.url) {
|
1484
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1485
|
+
}
|
1486
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1487
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1488
|
+
}
|
1489
|
+
} catch (e) {
|
1490
|
+
}
|
1491
|
+
}
|
1492
|
+
/**
|
1493
|
+
* Hafızayı temizler
|
1494
|
+
*/
|
1495
|
+
clearMemory() {
|
1496
|
+
if (this.persistentExecutor) {
|
1497
|
+
this.persistentExecutor.clearMemory();
|
1498
|
+
}
|
1499
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1500
|
+
}
|
1501
|
+
/**
|
1502
|
+
* Mevcut hafızayı döndürür
|
1503
|
+
*/
|
1504
|
+
getMemory() {
|
1505
|
+
return this.persistentExecutor?.getMemory() || [];
|
1506
|
+
}
|
1507
|
+
/**
|
1508
|
+
* İş akışı hafızasını döndürür
|
1509
|
+
*/
|
1510
|
+
getWorkflowMemory() {
|
1511
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1512
|
+
}
|
1513
|
+
/**
|
1514
|
+
* Hafıza istatistiklerini döndürür
|
1515
|
+
*/
|
1516
|
+
getMemoryStats() {
|
1517
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1518
|
+
totalItems: 0,
|
1519
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1520
|
+
config: this.memoryConfig
|
1521
|
+
};
|
1522
|
+
}
|
1523
|
+
/**
|
1524
|
+
* Hafıza konfigürasyonunu günceller
|
1525
|
+
*/
|
1526
|
+
updateMemoryConfig(config) {
|
1527
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1528
|
+
if (this.persistentExecutor) {
|
1529
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1530
|
+
}
|
1531
|
+
}
|
1532
|
+
/**
|
1533
|
+
* Oturum ID'sini oluşturur
|
1534
|
+
*/
|
1535
|
+
generateSessionId() {
|
1536
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1537
|
+
}
|
1538
|
+
/**
|
1539
|
+
* Hafızayı bağlam olarak formatlar
|
1540
|
+
*/
|
1541
|
+
getMemoryAsContext() {
|
1542
|
+
const memory = this.getMemory();
|
1543
|
+
if (memory.length === 0) {
|
1544
|
+
return "";
|
1545
|
+
}
|
1546
|
+
const recentMemory = memory.slice(-5);
|
1547
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1548
|
+
}
|
1312
1549
|
async runPlans(title, plans, opts) {
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1550
|
+
await this.updatePageContext();
|
1551
|
+
const useMemory = opts?.useMemory !== false;
|
1552
|
+
let taskExecutor;
|
1553
|
+
if (useMemory) {
|
1554
|
+
taskExecutor = this.getPersistentExecutor();
|
1555
|
+
this.workflowMemory.updateWorkflowContext({
|
1556
|
+
currentStep: title,
|
1557
|
+
pageInfo: this.sessionContext.pageInfo,
|
1558
|
+
timestamp: Date.now()
|
1559
|
+
}, this.sessionContext.workflowId || "default");
|
1560
|
+
} else {
|
1561
|
+
taskExecutor = new import_misoai_core.Executor(title, {
|
1562
|
+
onTaskStart: this.onTaskStartCallback
|
1563
|
+
});
|
1564
|
+
}
|
1316
1565
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1566
|
+
tasks.forEach((task) => {
|
1567
|
+
task.context = {
|
1568
|
+
...task.context,
|
1569
|
+
...this.sessionContext.pageInfo,
|
1570
|
+
workflowId: this.sessionContext.workflowId,
|
1571
|
+
sessionId: this.sessionContext.sessionId
|
1572
|
+
};
|
1573
|
+
});
|
1317
1574
|
await taskExecutor.append(tasks);
|
1318
1575
|
const result = await taskExecutor.flush();
|
1576
|
+
if (useMemory) {
|
1577
|
+
this.workflowMemory.saveWorkflowMemory(
|
1578
|
+
taskExecutor.getMemory(),
|
1579
|
+
this.sessionContext.workflowId || "default"
|
1580
|
+
);
|
1581
|
+
}
|
1319
1582
|
return {
|
1320
1583
|
output: result,
|
1321
1584
|
executor: taskExecutor
|
1322
1585
|
};
|
1323
1586
|
}
|
1324
1587
|
async action(userPrompt, actionContext, opts) {
|
1325
|
-
const
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1588
|
+
const useMemory = true;
|
1589
|
+
let taskExecutor;
|
1590
|
+
if (useMemory) {
|
1591
|
+
taskExecutor = this.getPersistentExecutor();
|
1592
|
+
} else {
|
1593
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1594
|
+
onTaskStart: this.onTaskStartCallback
|
1595
|
+
});
|
1596
|
+
}
|
1597
|
+
const memoryContext = this.getMemoryAsContext();
|
1598
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1599
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1329
1600
|
let replanCount = 0;
|
1330
1601
|
const logList = [];
|
1602
|
+
if (memoryContext) {
|
1603
|
+
logList.push(memoryContext);
|
1604
|
+
}
|
1331
1605
|
const yamlFlow = [];
|
1332
1606
|
while (planningTask) {
|
1333
1607
|
if (replanCount > replanningCountLimit) {
|
@@ -1436,16 +1710,22 @@ var PageTaskExecutor = class {
|
|
1436
1710
|
executor: taskExecutor
|
1437
1711
|
};
|
1438
1712
|
}
|
1439
|
-
async createTypeQueryTask(type, demand) {
|
1440
|
-
const
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1713
|
+
async createTypeQueryTask(type, demand, opt) {
|
1714
|
+
const useMemory = true;
|
1715
|
+
let taskExecutor;
|
1716
|
+
if (useMemory) {
|
1717
|
+
taskExecutor = this.getPersistentExecutor();
|
1718
|
+
} else {
|
1719
|
+
taskExecutor = new import_misoai_core.Executor(
|
1720
|
+
taskTitleStr(
|
1721
|
+
type,
|
1722
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1723
|
+
),
|
1724
|
+
{
|
1725
|
+
onTaskStart: this.onTaskStartCallback
|
1726
|
+
}
|
1727
|
+
);
|
1728
|
+
}
|
1449
1729
|
const queryTask = {
|
1450
1730
|
type: "Insight",
|
1451
1731
|
subType: type,
|
@@ -1467,7 +1747,13 @@ var PageTaskExecutor = class {
|
|
1467
1747
|
result: `${type}, ${demand}`
|
1468
1748
|
};
|
1469
1749
|
}
|
1470
|
-
const
|
1750
|
+
const memoryContext = this.getMemoryAsContext();
|
1751
|
+
const { data, usage } = await this.insight.extract(
|
1752
|
+
demandInput,
|
1753
|
+
opt,
|
1754
|
+
memoryContext
|
1755
|
+
// Hafıza bağlamını geç
|
1756
|
+
);
|
1471
1757
|
let outputResult = data;
|
1472
1758
|
if (ifTypeRestricted) {
|
1473
1759
|
(0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
|
@@ -1487,23 +1773,29 @@ var PageTaskExecutor = class {
|
|
1487
1773
|
executor: taskExecutor
|
1488
1774
|
};
|
1489
1775
|
}
|
1490
|
-
async query(demand) {
|
1491
|
-
return this.createTypeQueryTask("Query", demand);
|
1776
|
+
async query(demand, opt) {
|
1777
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1492
1778
|
}
|
1493
|
-
async boolean(prompt) {
|
1494
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1779
|
+
async boolean(prompt, opt) {
|
1780
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1495
1781
|
}
|
1496
|
-
async number(prompt) {
|
1497
|
-
return this.createTypeQueryTask("Number", prompt);
|
1782
|
+
async number(prompt, opt) {
|
1783
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1498
1784
|
}
|
1499
|
-
async string(prompt) {
|
1500
|
-
return this.createTypeQueryTask("String", prompt);
|
1785
|
+
async string(prompt, opt) {
|
1786
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1501
1787
|
}
|
1502
|
-
async assert(assertion) {
|
1788
|
+
async assert(assertion, memoryContext) {
|
1503
1789
|
const description = `assert: ${assertion}`;
|
1504
|
-
const
|
1505
|
-
|
1506
|
-
|
1790
|
+
const useMemory = true;
|
1791
|
+
let taskExecutor;
|
1792
|
+
if (useMemory) {
|
1793
|
+
taskExecutor = this.getPersistentExecutor();
|
1794
|
+
} else {
|
1795
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
|
1796
|
+
onTaskStart: this.onTaskStartCallback
|
1797
|
+
});
|
1798
|
+
}
|
1507
1799
|
const assertionPlan = {
|
1508
1800
|
type: "Assert",
|
1509
1801
|
param: {
|
@@ -1633,7 +1925,7 @@ function buildPlans(type, locateParam, param) {
|
|
1633
1925
|
param: locateParam,
|
1634
1926
|
thought: ""
|
1635
1927
|
} : null;
|
1636
|
-
if (type === "Tap" || type === "Hover") {
|
1928
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1637
1929
|
(0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
|
1638
1930
|
(0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
|
1639
1931
|
const tapPlan = {
|
@@ -1713,7 +2005,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1713
2005
|
var import_semver = __toESM(require("semver"));
|
1714
2006
|
|
1715
2007
|
// package.json
|
1716
|
-
var version = "1.5.
|
2008
|
+
var version = "1.5.7";
|
1717
2009
|
|
1718
2010
|
// src/common/task-cache.ts
|
1719
2011
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1844,8 +2136,14 @@ cache file: ${cacheFile}`
|
|
1844
2136
|
return;
|
1845
2137
|
}
|
1846
2138
|
try {
|
2139
|
+
const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
|
2140
|
+
if (!(0, import_node_fs2.existsSync)(dir)) {
|
2141
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
2142
|
+
debug3("created cache directory: %s", dir);
|
2143
|
+
}
|
1847
2144
|
const yamlData = import_js_yaml3.default.dump(this.cache);
|
1848
2145
|
(0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
|
2146
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1849
2147
|
} catch (err) {
|
1850
2148
|
debug3(
|
1851
2149
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2103,6 +2401,23 @@ var PageAgent = class {
|
|
2103
2401
|
metadata
|
2104
2402
|
};
|
2105
2403
|
}
|
2404
|
+
async aiRightClick(locatePrompt, opt) {
|
2405
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2406
|
+
locatePrompt,
|
2407
|
+
opt
|
2408
|
+
);
|
2409
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2410
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2411
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2412
|
+
plans,
|
2413
|
+
{ cacheable: opt?.cacheable }
|
2414
|
+
);
|
2415
|
+
const metadata = this.afterTaskRunning(executor);
|
2416
|
+
return {
|
2417
|
+
result: output,
|
2418
|
+
metadata
|
2419
|
+
};
|
2420
|
+
}
|
2106
2421
|
async aiInput(value, locatePrompt, opt) {
|
2107
2422
|
(0, import_utils12.assert)(
|
2108
2423
|
typeof value === "string",
|
@@ -2177,7 +2492,13 @@ var PageAgent = class {
|
|
2177
2492
|
metadata: metadata2
|
2178
2493
|
};
|
2179
2494
|
}
|
2180
|
-
const
|
2495
|
+
const memoryContext = this.getMemoryAsContext();
|
2496
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2497
|
+
|
2498
|
+
Previous workflow steps:
|
2499
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2500
|
+
${memoryContext}` : void 0;
|
2501
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2181
2502
|
cacheable
|
2182
2503
|
}));
|
2183
2504
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2323,8 +2644,9 @@ var PageAgent = class {
|
|
2323
2644
|
} catch (e) {
|
2324
2645
|
}
|
2325
2646
|
}
|
2647
|
+
const memoryContext = this.getMemoryAsContext();
|
2326
2648
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2327
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2649
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2328
2650
|
const metadata = this.afterTaskRunning(executor, true);
|
2329
2651
|
if (output && opt?.keepRawResponse) {
|
2330
2652
|
return {
|
@@ -2534,9 +2856,72 @@ ${errors}`);
|
|
2534
2856
|
}
|
2535
2857
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2536
2858
|
}
|
2859
|
+
async logScreenshot(title, options) {
|
2860
|
+
const screenshotTitle = title || "untitled";
|
2861
|
+
const content = options?.content || "";
|
2862
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2863
|
+
if (screenshot) {
|
2864
|
+
const executionDump = {
|
2865
|
+
name: screenshotTitle,
|
2866
|
+
description: content,
|
2867
|
+
tasks: [{
|
2868
|
+
type: "Screenshot",
|
2869
|
+
subType: "log",
|
2870
|
+
status: "finished",
|
2871
|
+
executor: null,
|
2872
|
+
param: {
|
2873
|
+
title: screenshotTitle,
|
2874
|
+
content
|
2875
|
+
},
|
2876
|
+
output: {
|
2877
|
+
screenshot
|
2878
|
+
},
|
2879
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2880
|
+
timing: {
|
2881
|
+
start: Date.now(),
|
2882
|
+
end: Date.now(),
|
2883
|
+
cost: 0
|
2884
|
+
}
|
2885
|
+
}],
|
2886
|
+
sdkVersion: "1.0.0",
|
2887
|
+
logTime: Date.now(),
|
2888
|
+
model_name: "screenshot"
|
2889
|
+
};
|
2890
|
+
this.appendExecutionDump(executionDump);
|
2891
|
+
}
|
2892
|
+
}
|
2537
2893
|
async destroy() {
|
2538
2894
|
await this.page.destroy();
|
2539
2895
|
}
|
2896
|
+
/**
|
2897
|
+
* Hafızayı bağlam olarak formatlar
|
2898
|
+
*/
|
2899
|
+
getMemoryAsContext() {
|
2900
|
+
const memory = this.taskExecutor.getMemory();
|
2901
|
+
if (memory.length === 0) {
|
2902
|
+
return "";
|
2903
|
+
}
|
2904
|
+
const recentMemory = memory.slice(-5);
|
2905
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2906
|
+
}
|
2907
|
+
/**
|
2908
|
+
* Mevcut hafızayı döndürür
|
2909
|
+
*/
|
2910
|
+
getMemory() {
|
2911
|
+
return this.taskExecutor.getMemory();
|
2912
|
+
}
|
2913
|
+
/**
|
2914
|
+
* Hafıza istatistiklerini döndürür
|
2915
|
+
*/
|
2916
|
+
getMemoryStats() {
|
2917
|
+
return this.taskExecutor.getMemoryStats();
|
2918
|
+
}
|
2919
|
+
/**
|
2920
|
+
* Hafızayı temizler
|
2921
|
+
*/
|
2922
|
+
clearMemory() {
|
2923
|
+
this.taskExecutor.clearMemory();
|
2924
|
+
}
|
2540
2925
|
};
|
2541
2926
|
|
2542
2927
|
// src/puppeteer/base-page.ts
|
@@ -2957,6 +3342,14 @@ var PlaywrightAiFixture = (options) => {
|
|
2957
3342
|
aiActionType: "aiTap"
|
2958
3343
|
});
|
2959
3344
|
},
|
3345
|
+
aiRightClick: async ({ page }, use, testInfo) => {
|
3346
|
+
await generateAiFunction({
|
3347
|
+
page,
|
3348
|
+
testInfo,
|
3349
|
+
use,
|
3350
|
+
aiActionType: "aiRightClick"
|
3351
|
+
});
|
3352
|
+
},
|
2960
3353
|
aiHover: async ({ page }, use, testInfo) => {
|
2961
3354
|
await generateAiFunction({
|
2962
3355
|
page,
|
@@ -3044,6 +3437,14 @@ var PlaywrightAiFixture = (options) => {
|
|
3044
3437
|
use,
|
3045
3438
|
aiActionType: "aiBoolean"
|
3046
3439
|
});
|
3440
|
+
},
|
3441
|
+
logScreenshot: async ({ page }, use, testInfo) => {
|
3442
|
+
await generateAiFunction({
|
3443
|
+
page,
|
3444
|
+
testInfo,
|
3445
|
+
use,
|
3446
|
+
aiActionType: "logScreenshot"
|
3447
|
+
});
|
3047
3448
|
}
|
3048
3449
|
};
|
3049
3450
|
};
|