misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
package/dist/lib/playwright.js
CHANGED
@@ -144,6 +144,10 @@ var ScriptPlayer = class {
|
|
144
144
|
} else if ("aiQuery" in flowItem) {
|
145
145
|
const queryTask = flowItem;
|
146
146
|
const prompt = queryTask.aiQuery;
|
147
|
+
const options = {
|
148
|
+
domIncluded: queryTask.domIncluded,
|
149
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
150
|
+
};
|
147
151
|
(0, import_utils.assert)(prompt, "missing prompt for aiQuery");
|
148
152
|
(0, import_utils.assert)(
|
149
153
|
typeof prompt === "string",
|
@@ -154,6 +158,10 @@ var ScriptPlayer = class {
|
|
154
158
|
} else if ("aiNumber" in flowItem) {
|
155
159
|
const numberTask = flowItem;
|
156
160
|
const prompt = numberTask.aiNumber;
|
161
|
+
const options = {
|
162
|
+
domIncluded: numberTask.domIncluded,
|
163
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
164
|
+
};
|
157
165
|
(0, import_utils.assert)(prompt, "missing prompt for number");
|
158
166
|
(0, import_utils.assert)(
|
159
167
|
typeof prompt === "string",
|
@@ -164,6 +172,10 @@ var ScriptPlayer = class {
|
|
164
172
|
} else if ("aiString" in flowItem) {
|
165
173
|
const stringTask = flowItem;
|
166
174
|
const prompt = stringTask.aiString;
|
175
|
+
const options = {
|
176
|
+
domIncluded: stringTask.domIncluded,
|
177
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
178
|
+
};
|
167
179
|
(0, import_utils.assert)(prompt, "missing prompt for string");
|
168
180
|
(0, import_utils.assert)(
|
169
181
|
typeof prompt === "string",
|
@@ -174,6 +186,10 @@ var ScriptPlayer = class {
|
|
174
186
|
} else if ("aiBoolean" in flowItem) {
|
175
187
|
const booleanTask = flowItem;
|
176
188
|
const prompt = booleanTask.aiBoolean;
|
189
|
+
const options = {
|
190
|
+
domIncluded: booleanTask.domIncluded,
|
191
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
192
|
+
};
|
177
193
|
(0, import_utils.assert)(prompt, "missing prompt for boolean");
|
178
194
|
(0, import_utils.assert)(
|
179
195
|
typeof prompt === "string",
|
@@ -216,6 +232,9 @@ var ScriptPlayer = class {
|
|
216
232
|
} else if ("aiTap" in flowItem) {
|
217
233
|
const tapTask = flowItem;
|
218
234
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
235
|
+
} else if ("aiRightClick" in flowItem) {
|
236
|
+
const rightClickTask = flowItem;
|
237
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
219
238
|
} else if ("aiHover" in flowItem) {
|
220
239
|
const hoverTask = flowItem;
|
221
240
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -238,6 +257,11 @@ var ScriptPlayer = class {
|
|
238
257
|
evaluateJavaScriptTask.javascript
|
239
258
|
);
|
240
259
|
this.setResult(evaluateJavaScriptTask.name, result);
|
260
|
+
} else if ("logScreenshot" in flowItem) {
|
261
|
+
const logScreenshotTask = flowItem;
|
262
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
263
|
+
content: logScreenshotTask.content || ""
|
264
|
+
});
|
241
265
|
} else {
|
242
266
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
243
267
|
}
|
@@ -486,7 +510,8 @@ var WebElementInfo = class {
|
|
486
510
|
id,
|
487
511
|
attributes,
|
488
512
|
indexId,
|
489
|
-
xpaths
|
513
|
+
xpaths,
|
514
|
+
isVisible
|
490
515
|
}) {
|
491
516
|
this.content = content;
|
492
517
|
this.rect = rect;
|
@@ -499,6 +524,7 @@ var WebElementInfo = class {
|
|
499
524
|
this.attributes = attributes;
|
500
525
|
this.indexId = indexId;
|
501
526
|
this.xpaths = xpaths;
|
527
|
+
this.isVisible = isVisible;
|
502
528
|
}
|
503
529
|
};
|
504
530
|
|
@@ -521,14 +547,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
521
547
|
})
|
522
548
|
]);
|
523
549
|
const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
|
524
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
550
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
525
551
|
return new WebElementInfo({
|
526
552
|
rect,
|
527
553
|
locator,
|
528
554
|
id,
|
529
555
|
content,
|
530
556
|
attributes,
|
531
|
-
indexId
|
557
|
+
indexId,
|
558
|
+
isVisible
|
532
559
|
});
|
533
560
|
});
|
534
561
|
(0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
|
@@ -558,7 +585,7 @@ function printReportMsg(filepath) {
|
|
558
585
|
(0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
|
559
586
|
}
|
560
587
|
function replaceIllegalPathCharsAndSpace(str) {
|
561
|
-
return str.replace(/[
|
588
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
562
589
|
}
|
563
590
|
function forceClosePopup(page, debug6) {
|
564
591
|
page.on("popup", async (popup) => {
|
@@ -615,6 +642,94 @@ var replanningCountLimit = 10;
|
|
615
642
|
var isAndroidPage = (page) => {
|
616
643
|
return page.pageType === "android";
|
617
644
|
};
|
645
|
+
var WorkflowMemory = class {
|
646
|
+
constructor(config) {
|
647
|
+
this.workflows = /* @__PURE__ */ new Map();
|
648
|
+
this.config = config;
|
649
|
+
}
|
650
|
+
/**
|
651
|
+
* İş akışı hafızasını getirir
|
652
|
+
*/
|
653
|
+
getWorkflowMemory(workflowId = "default") {
|
654
|
+
const workflow = this.workflows.get(workflowId);
|
655
|
+
return workflow?.memory || [];
|
656
|
+
}
|
657
|
+
/**
|
658
|
+
* İş akışı verilerini getirir
|
659
|
+
*/
|
660
|
+
getWorkflowData(workflowId = "default") {
|
661
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
662
|
+
}
|
663
|
+
/**
|
664
|
+
* İş akışı hafızasını kaydeder
|
665
|
+
*/
|
666
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
667
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
668
|
+
workflow.memory = [...memory];
|
669
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
670
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
671
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
672
|
+
this.workflows.set(workflowId, workflow);
|
673
|
+
this.enforceRetentionPolicy();
|
674
|
+
}
|
675
|
+
/**
|
676
|
+
* İş akışı bağlamını günceller
|
677
|
+
*/
|
678
|
+
updateWorkflowContext(context, workflowId = "default") {
|
679
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
680
|
+
workflow.context = { ...workflow.context, ...context };
|
681
|
+
if (context.currentStep) {
|
682
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
683
|
+
if (!existingStep) {
|
684
|
+
workflow.steps.push({
|
685
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
686
|
+
stepName: context.currentStep,
|
687
|
+
timestamp: context.timestamp,
|
688
|
+
status: "running",
|
689
|
+
memoryItems: []
|
690
|
+
});
|
691
|
+
}
|
692
|
+
}
|
693
|
+
this.workflows.set(workflowId, workflow);
|
694
|
+
}
|
695
|
+
/**
|
696
|
+
* İş akışını temizler
|
697
|
+
*/
|
698
|
+
clearWorkflow(workflowId = "default") {
|
699
|
+
this.workflows.delete(workflowId);
|
700
|
+
}
|
701
|
+
/**
|
702
|
+
* Tüm iş akışlarını temizler
|
703
|
+
*/
|
704
|
+
clearAll() {
|
705
|
+
this.workflows.clear();
|
706
|
+
}
|
707
|
+
createEmptyWorkflowData(workflowId) {
|
708
|
+
return {
|
709
|
+
workflowId,
|
710
|
+
steps: [],
|
711
|
+
memory: [],
|
712
|
+
context: {
|
713
|
+
pageInfo: { url: "", title: "" },
|
714
|
+
timestamp: Date.now()
|
715
|
+
},
|
716
|
+
metadata: {
|
717
|
+
totalSteps: 0,
|
718
|
+
completedSteps: 0,
|
719
|
+
failedSteps: 0,
|
720
|
+
startTime: Date.now()
|
721
|
+
}
|
722
|
+
};
|
723
|
+
}
|
724
|
+
enforceRetentionPolicy() {
|
725
|
+
const maxWorkflows = 10;
|
726
|
+
if (this.workflows.size > maxWorkflows) {
|
727
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
728
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
729
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
730
|
+
}
|
731
|
+
}
|
732
|
+
};
|
618
733
|
var PageTaskExecutor = class {
|
619
734
|
constructor(page, insight, opts) {
|
620
735
|
this.conversationHistory = [];
|
@@ -622,6 +737,25 @@ var PageTaskExecutor = class {
|
|
622
737
|
this.insight = insight;
|
623
738
|
this.taskCache = opts.taskCache;
|
624
739
|
this.onTaskStartCallback = opts?.onTaskStart;
|
740
|
+
this.memoryConfig = {
|
741
|
+
maxItems: 100,
|
742
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
743
|
+
// 2 saat
|
744
|
+
enablePersistence: true,
|
745
|
+
enableAnalytics: true,
|
746
|
+
filterStrategy: "hybrid",
|
747
|
+
...opts?.memoryConfig
|
748
|
+
};
|
749
|
+
this.sessionContext = {
|
750
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
751
|
+
workflowId: opts?.workflowId,
|
752
|
+
startTime: Date.now(),
|
753
|
+
pageInfo: {
|
754
|
+
url: "",
|
755
|
+
title: ""
|
756
|
+
}
|
757
|
+
};
|
758
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
625
759
|
}
|
626
760
|
async recordScreenshot(timing) {
|
627
761
|
const base64 = await this.page.screenshotBase64();
|
@@ -834,8 +968,11 @@ var PageTaskExecutor = class {
|
|
834
968
|
insightDump = dump;
|
835
969
|
};
|
836
970
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
971
|
+
const memoryContext = this.getMemoryAsContext();
|
837
972
|
const assertion = await this.insight.assert(
|
838
|
-
assertPlan.param.assertion
|
973
|
+
assertPlan.param.assertion,
|
974
|
+
memoryContext
|
975
|
+
// Hafıza bağlamını geç
|
839
976
|
);
|
840
977
|
if (!assertion.pass) {
|
841
978
|
if (plan2.type === "Assert") {
|
@@ -872,10 +1009,10 @@ var PageTaskExecutor = class {
|
|
872
1009
|
if (!taskParam || !taskParam.value) {
|
873
1010
|
return;
|
874
1011
|
}
|
875
|
-
await this.page.keyboard.type(taskParam.value);
|
876
|
-
} else {
|
877
|
-
await this.page.keyboard.type(taskParam.value);
|
878
1012
|
}
|
1013
|
+
await this.page.keyboard.type(taskParam.value, {
|
1014
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
1015
|
+
});
|
879
1016
|
}
|
880
1017
|
};
|
881
1018
|
tasks.push(taskActionInput);
|
@@ -904,6 +1041,22 @@ var PageTaskExecutor = class {
|
|
904
1041
|
}
|
905
1042
|
};
|
906
1043
|
tasks.push(taskActionTap);
|
1044
|
+
} else if (plan2.type === "RightClick") {
|
1045
|
+
const taskActionRightClick = {
|
1046
|
+
type: "Action",
|
1047
|
+
subType: "RightClick",
|
1048
|
+
thought: plan2.thought,
|
1049
|
+
locate: plan2.locate,
|
1050
|
+
executor: async (param, { element }) => {
|
1051
|
+
(0, import_utils6.assert)(element, "Element not found, cannot right click");
|
1052
|
+
await this.page.mouse.click(
|
1053
|
+
element.center[0],
|
1054
|
+
element.center[1],
|
1055
|
+
{ button: "right" }
|
1056
|
+
);
|
1057
|
+
}
|
1058
|
+
};
|
1059
|
+
tasks.push(taskActionRightClick);
|
907
1060
|
} else if (plan2.type === "Drag") {
|
908
1061
|
const taskActionDrag = {
|
909
1062
|
type: "Action",
|
@@ -1305,25 +1458,146 @@ var PageTaskExecutor = class {
|
|
1305
1458
|
};
|
1306
1459
|
return task;
|
1307
1460
|
}
|
1461
|
+
/**
|
1462
|
+
* Persistent executor'ı getirir veya oluşturur
|
1463
|
+
*/
|
1464
|
+
getPersistentExecutor() {
|
1465
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1466
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1467
|
+
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1468
|
+
onTaskStart: this.onTaskStartCallback,
|
1469
|
+
initialMemory: previousMemory
|
1470
|
+
});
|
1471
|
+
}
|
1472
|
+
return this.persistentExecutor;
|
1473
|
+
}
|
1474
|
+
/**
|
1475
|
+
* Sayfa bağlamını günceller
|
1476
|
+
*/
|
1477
|
+
async updatePageContext() {
|
1478
|
+
try {
|
1479
|
+
if (this.page.url) {
|
1480
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1481
|
+
}
|
1482
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1483
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1484
|
+
}
|
1485
|
+
} catch (e) {
|
1486
|
+
}
|
1487
|
+
}
|
1488
|
+
/**
|
1489
|
+
* Hafızayı temizler
|
1490
|
+
*/
|
1491
|
+
clearMemory() {
|
1492
|
+
if (this.persistentExecutor) {
|
1493
|
+
this.persistentExecutor.clearMemory();
|
1494
|
+
}
|
1495
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1496
|
+
}
|
1497
|
+
/**
|
1498
|
+
* Mevcut hafızayı döndürür
|
1499
|
+
*/
|
1500
|
+
getMemory() {
|
1501
|
+
return this.persistentExecutor?.getMemory() || [];
|
1502
|
+
}
|
1503
|
+
/**
|
1504
|
+
* İş akışı hafızasını döndürür
|
1505
|
+
*/
|
1506
|
+
getWorkflowMemory() {
|
1507
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1508
|
+
}
|
1509
|
+
/**
|
1510
|
+
* Hafıza istatistiklerini döndürür
|
1511
|
+
*/
|
1512
|
+
getMemoryStats() {
|
1513
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1514
|
+
totalItems: 0,
|
1515
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1516
|
+
config: this.memoryConfig
|
1517
|
+
};
|
1518
|
+
}
|
1519
|
+
/**
|
1520
|
+
* Hafıza konfigürasyonunu günceller
|
1521
|
+
*/
|
1522
|
+
updateMemoryConfig(config) {
|
1523
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1524
|
+
if (this.persistentExecutor) {
|
1525
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1526
|
+
}
|
1527
|
+
}
|
1528
|
+
/**
|
1529
|
+
* Oturum ID'sini oluşturur
|
1530
|
+
*/
|
1531
|
+
generateSessionId() {
|
1532
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1533
|
+
}
|
1534
|
+
/**
|
1535
|
+
* Hafızayı bağlam olarak formatlar
|
1536
|
+
*/
|
1537
|
+
getMemoryAsContext() {
|
1538
|
+
const memory = this.getMemory();
|
1539
|
+
if (memory.length === 0) {
|
1540
|
+
return "";
|
1541
|
+
}
|
1542
|
+
const recentMemory = memory.slice(-5);
|
1543
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1544
|
+
}
|
1308
1545
|
async runPlans(title, plans, opts) {
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1546
|
+
await this.updatePageContext();
|
1547
|
+
const useMemory = opts?.useMemory !== false;
|
1548
|
+
let taskExecutor;
|
1549
|
+
if (useMemory) {
|
1550
|
+
taskExecutor = this.getPersistentExecutor();
|
1551
|
+
this.workflowMemory.updateWorkflowContext({
|
1552
|
+
currentStep: title,
|
1553
|
+
pageInfo: this.sessionContext.pageInfo,
|
1554
|
+
timestamp: Date.now()
|
1555
|
+
}, this.sessionContext.workflowId || "default");
|
1556
|
+
} else {
|
1557
|
+
taskExecutor = new import_misoai_core.Executor(title, {
|
1558
|
+
onTaskStart: this.onTaskStartCallback
|
1559
|
+
});
|
1560
|
+
}
|
1312
1561
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1562
|
+
tasks.forEach((task) => {
|
1563
|
+
task.context = {
|
1564
|
+
...task.context,
|
1565
|
+
...this.sessionContext.pageInfo,
|
1566
|
+
workflowId: this.sessionContext.workflowId,
|
1567
|
+
sessionId: this.sessionContext.sessionId
|
1568
|
+
};
|
1569
|
+
});
|
1313
1570
|
await taskExecutor.append(tasks);
|
1314
1571
|
const result = await taskExecutor.flush();
|
1572
|
+
if (useMemory) {
|
1573
|
+
this.workflowMemory.saveWorkflowMemory(
|
1574
|
+
taskExecutor.getMemory(),
|
1575
|
+
this.sessionContext.workflowId || "default"
|
1576
|
+
);
|
1577
|
+
}
|
1315
1578
|
return {
|
1316
1579
|
output: result,
|
1317
1580
|
executor: taskExecutor
|
1318
1581
|
};
|
1319
1582
|
}
|
1320
1583
|
async action(userPrompt, actionContext, opts) {
|
1321
|
-
const
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1584
|
+
const useMemory = true;
|
1585
|
+
let taskExecutor;
|
1586
|
+
if (useMemory) {
|
1587
|
+
taskExecutor = this.getPersistentExecutor();
|
1588
|
+
} else {
|
1589
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1590
|
+
onTaskStart: this.onTaskStartCallback
|
1591
|
+
});
|
1592
|
+
}
|
1593
|
+
const memoryContext = this.getMemoryAsContext();
|
1594
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1595
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1325
1596
|
let replanCount = 0;
|
1326
1597
|
const logList = [];
|
1598
|
+
if (memoryContext) {
|
1599
|
+
logList.push(memoryContext);
|
1600
|
+
}
|
1327
1601
|
const yamlFlow = [];
|
1328
1602
|
while (planningTask) {
|
1329
1603
|
if (replanCount > replanningCountLimit) {
|
@@ -1432,16 +1706,22 @@ var PageTaskExecutor = class {
|
|
1432
1706
|
executor: taskExecutor
|
1433
1707
|
};
|
1434
1708
|
}
|
1435
|
-
async createTypeQueryTask(type, demand) {
|
1436
|
-
const
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1709
|
+
async createTypeQueryTask(type, demand, opt) {
|
1710
|
+
const useMemory = true;
|
1711
|
+
let taskExecutor;
|
1712
|
+
if (useMemory) {
|
1713
|
+
taskExecutor = this.getPersistentExecutor();
|
1714
|
+
} else {
|
1715
|
+
taskExecutor = new import_misoai_core.Executor(
|
1716
|
+
taskTitleStr(
|
1717
|
+
type,
|
1718
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1719
|
+
),
|
1720
|
+
{
|
1721
|
+
onTaskStart: this.onTaskStartCallback
|
1722
|
+
}
|
1723
|
+
);
|
1724
|
+
}
|
1445
1725
|
const queryTask = {
|
1446
1726
|
type: "Insight",
|
1447
1727
|
subType: type,
|
@@ -1463,7 +1743,13 @@ var PageTaskExecutor = class {
|
|
1463
1743
|
result: `${type}, ${demand}`
|
1464
1744
|
};
|
1465
1745
|
}
|
1466
|
-
const
|
1746
|
+
const memoryContext = this.getMemoryAsContext();
|
1747
|
+
const { data, usage } = await this.insight.extract(
|
1748
|
+
demandInput,
|
1749
|
+
opt,
|
1750
|
+
memoryContext
|
1751
|
+
// Hafıza bağlamını geç
|
1752
|
+
);
|
1467
1753
|
let outputResult = data;
|
1468
1754
|
if (ifTypeRestricted) {
|
1469
1755
|
(0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
|
@@ -1483,23 +1769,29 @@ var PageTaskExecutor = class {
|
|
1483
1769
|
executor: taskExecutor
|
1484
1770
|
};
|
1485
1771
|
}
|
1486
|
-
async query(demand) {
|
1487
|
-
return this.createTypeQueryTask("Query", demand);
|
1772
|
+
async query(demand, opt) {
|
1773
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1488
1774
|
}
|
1489
|
-
async boolean(prompt) {
|
1490
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1775
|
+
async boolean(prompt, opt) {
|
1776
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1491
1777
|
}
|
1492
|
-
async number(prompt) {
|
1493
|
-
return this.createTypeQueryTask("Number", prompt);
|
1778
|
+
async number(prompt, opt) {
|
1779
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1494
1780
|
}
|
1495
|
-
async string(prompt) {
|
1496
|
-
return this.createTypeQueryTask("String", prompt);
|
1781
|
+
async string(prompt, opt) {
|
1782
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1497
1783
|
}
|
1498
|
-
async assert(assertion) {
|
1784
|
+
async assert(assertion, memoryContext) {
|
1499
1785
|
const description = `assert: ${assertion}`;
|
1500
|
-
const
|
1501
|
-
|
1502
|
-
|
1786
|
+
const useMemory = true;
|
1787
|
+
let taskExecutor;
|
1788
|
+
if (useMemory) {
|
1789
|
+
taskExecutor = this.getPersistentExecutor();
|
1790
|
+
} else {
|
1791
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
|
1792
|
+
onTaskStart: this.onTaskStartCallback
|
1793
|
+
});
|
1794
|
+
}
|
1503
1795
|
const assertionPlan = {
|
1504
1796
|
type: "Assert",
|
1505
1797
|
param: {
|
@@ -1629,7 +1921,7 @@ function buildPlans(type, locateParam, param) {
|
|
1629
1921
|
param: locateParam,
|
1630
1922
|
thought: ""
|
1631
1923
|
} : null;
|
1632
|
-
if (type === "Tap" || type === "Hover") {
|
1924
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1633
1925
|
(0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
|
1634
1926
|
(0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
|
1635
1927
|
const tapPlan = {
|
@@ -1709,7 +2001,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1709
2001
|
var import_semver = __toESM(require("semver"));
|
1710
2002
|
|
1711
2003
|
// package.json
|
1712
|
-
var version = "1.5.
|
2004
|
+
var version = "1.5.7";
|
1713
2005
|
|
1714
2006
|
// src/common/task-cache.ts
|
1715
2007
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1840,8 +2132,14 @@ cache file: ${cacheFile}`
|
|
1840
2132
|
return;
|
1841
2133
|
}
|
1842
2134
|
try {
|
2135
|
+
const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
|
2136
|
+
if (!(0, import_node_fs2.existsSync)(dir)) {
|
2137
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
2138
|
+
debug3("created cache directory: %s", dir);
|
2139
|
+
}
|
1843
2140
|
const yamlData = import_js_yaml3.default.dump(this.cache);
|
1844
2141
|
(0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
|
2142
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1845
2143
|
} catch (err) {
|
1846
2144
|
debug3(
|
1847
2145
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2099,6 +2397,23 @@ var PageAgent = class {
|
|
2099
2397
|
metadata
|
2100
2398
|
};
|
2101
2399
|
}
|
2400
|
+
async aiRightClick(locatePrompt, opt) {
|
2401
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2402
|
+
locatePrompt,
|
2403
|
+
opt
|
2404
|
+
);
|
2405
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2406
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2407
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2408
|
+
plans,
|
2409
|
+
{ cacheable: opt?.cacheable }
|
2410
|
+
);
|
2411
|
+
const metadata = this.afterTaskRunning(executor);
|
2412
|
+
return {
|
2413
|
+
result: output,
|
2414
|
+
metadata
|
2415
|
+
};
|
2416
|
+
}
|
2102
2417
|
async aiInput(value, locatePrompt, opt) {
|
2103
2418
|
(0, import_utils12.assert)(
|
2104
2419
|
typeof value === "string",
|
@@ -2173,7 +2488,13 @@ var PageAgent = class {
|
|
2173
2488
|
metadata: metadata2
|
2174
2489
|
};
|
2175
2490
|
}
|
2176
|
-
const
|
2491
|
+
const memoryContext = this.getMemoryAsContext();
|
2492
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2493
|
+
|
2494
|
+
Previous workflow steps:
|
2495
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2496
|
+
${memoryContext}` : void 0;
|
2497
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2177
2498
|
cacheable
|
2178
2499
|
}));
|
2179
2500
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2319,8 +2640,9 @@ var PageAgent = class {
|
|
2319
2640
|
} catch (e) {
|
2320
2641
|
}
|
2321
2642
|
}
|
2643
|
+
const memoryContext = this.getMemoryAsContext();
|
2322
2644
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2323
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2645
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2324
2646
|
const metadata = this.afterTaskRunning(executor, true);
|
2325
2647
|
if (output && opt?.keepRawResponse) {
|
2326
2648
|
return {
|
@@ -2530,9 +2852,72 @@ ${errors}`);
|
|
2530
2852
|
}
|
2531
2853
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2532
2854
|
}
|
2855
|
+
async logScreenshot(title, options) {
|
2856
|
+
const screenshotTitle = title || "untitled";
|
2857
|
+
const content = options?.content || "";
|
2858
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2859
|
+
if (screenshot) {
|
2860
|
+
const executionDump = {
|
2861
|
+
name: screenshotTitle,
|
2862
|
+
description: content,
|
2863
|
+
tasks: [{
|
2864
|
+
type: "Screenshot",
|
2865
|
+
subType: "log",
|
2866
|
+
status: "finished",
|
2867
|
+
executor: null,
|
2868
|
+
param: {
|
2869
|
+
title: screenshotTitle,
|
2870
|
+
content
|
2871
|
+
},
|
2872
|
+
output: {
|
2873
|
+
screenshot
|
2874
|
+
},
|
2875
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2876
|
+
timing: {
|
2877
|
+
start: Date.now(),
|
2878
|
+
end: Date.now(),
|
2879
|
+
cost: 0
|
2880
|
+
}
|
2881
|
+
}],
|
2882
|
+
sdkVersion: "1.0.0",
|
2883
|
+
logTime: Date.now(),
|
2884
|
+
model_name: "screenshot"
|
2885
|
+
};
|
2886
|
+
this.appendExecutionDump(executionDump);
|
2887
|
+
}
|
2888
|
+
}
|
2533
2889
|
async destroy() {
|
2534
2890
|
await this.page.destroy();
|
2535
2891
|
}
|
2892
|
+
/**
|
2893
|
+
* Hafızayı bağlam olarak formatlar
|
2894
|
+
*/
|
2895
|
+
getMemoryAsContext() {
|
2896
|
+
const memory = this.taskExecutor.getMemory();
|
2897
|
+
if (memory.length === 0) {
|
2898
|
+
return "";
|
2899
|
+
}
|
2900
|
+
const recentMemory = memory.slice(-5);
|
2901
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2902
|
+
}
|
2903
|
+
/**
|
2904
|
+
* Mevcut hafızayı döndürür
|
2905
|
+
*/
|
2906
|
+
getMemory() {
|
2907
|
+
return this.taskExecutor.getMemory();
|
2908
|
+
}
|
2909
|
+
/**
|
2910
|
+
* Hafıza istatistiklerini döndürür
|
2911
|
+
*/
|
2912
|
+
getMemoryStats() {
|
2913
|
+
return this.taskExecutor.getMemoryStats();
|
2914
|
+
}
|
2915
|
+
/**
|
2916
|
+
* Hafızayı temizler
|
2917
|
+
*/
|
2918
|
+
clearMemory() {
|
2919
|
+
this.taskExecutor.clearMemory();
|
2920
|
+
}
|
2536
2921
|
};
|
2537
2922
|
|
2538
2923
|
// src/puppeteer/base-page.ts
|
@@ -2953,6 +3338,14 @@ var PlaywrightAiFixture = (options) => {
|
|
2953
3338
|
aiActionType: "aiTap"
|
2954
3339
|
});
|
2955
3340
|
},
|
3341
|
+
aiRightClick: async ({ page }, use, testInfo) => {
|
3342
|
+
await generateAiFunction({
|
3343
|
+
page,
|
3344
|
+
testInfo,
|
3345
|
+
use,
|
3346
|
+
aiActionType: "aiRightClick"
|
3347
|
+
});
|
3348
|
+
},
|
2956
3349
|
aiHover: async ({ page }, use, testInfo) => {
|
2957
3350
|
await generateAiFunction({
|
2958
3351
|
page,
|
@@ -3040,6 +3433,14 @@ var PlaywrightAiFixture = (options) => {
|
|
3040
3433
|
use,
|
3041
3434
|
aiActionType: "aiBoolean"
|
3042
3435
|
});
|
3436
|
+
},
|
3437
|
+
logScreenshot: async ({ page }, use, testInfo) => {
|
3438
|
+
await generateAiFunction({
|
3439
|
+
page,
|
3440
|
+
testInfo,
|
3441
|
+
use,
|
3442
|
+
aiActionType: "logScreenshot"
|
3443
|
+
});
|
3043
3444
|
}
|
3044
3445
|
};
|
3045
3446
|
};
|