misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
@@ -78,7 +78,8 @@ var WebElementInfo = class {
|
|
78
78
|
id,
|
79
79
|
attributes,
|
80
80
|
indexId,
|
81
|
-
xpaths
|
81
|
+
xpaths,
|
82
|
+
isVisible
|
82
83
|
}) {
|
83
84
|
this.content = content;
|
84
85
|
this.rect = rect;
|
@@ -91,6 +92,7 @@ var WebElementInfo = class {
|
|
91
92
|
this.attributes = attributes;
|
92
93
|
this.indexId = indexId;
|
93
94
|
this.xpaths = xpaths;
|
95
|
+
this.isVisible = isVisible;
|
94
96
|
}
|
95
97
|
};
|
96
98
|
|
@@ -113,14 +115,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
113
115
|
})
|
114
116
|
]);
|
115
117
|
const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
|
116
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
118
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
117
119
|
return new WebElementInfo({
|
118
120
|
rect,
|
119
121
|
locator,
|
120
122
|
id,
|
121
123
|
content,
|
122
124
|
attributes,
|
123
|
-
indexId
|
125
|
+
indexId,
|
126
|
+
isVisible
|
124
127
|
});
|
125
128
|
});
|
126
129
|
(0, import_utils2.assert)(screenshotBase64, "screenshotBase64 is required");
|
@@ -151,7 +154,7 @@ function printReportMsg(filepath) {
|
|
151
154
|
}
|
152
155
|
var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
|
153
156
|
function replaceIllegalPathCharsAndSpace(str) {
|
154
|
-
return str.replace(/[
|
157
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
155
158
|
}
|
156
159
|
function matchElementFromPlan(planLocateParam, tree) {
|
157
160
|
if (!planLocateParam) {
|
@@ -281,6 +284,10 @@ var ScriptPlayer = class {
|
|
281
284
|
} else if ("aiQuery" in flowItem) {
|
282
285
|
const queryTask = flowItem;
|
283
286
|
const prompt = queryTask.aiQuery;
|
287
|
+
const options = {
|
288
|
+
domIncluded: queryTask.domIncluded,
|
289
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
290
|
+
};
|
284
291
|
(0, import_utils3.assert)(prompt, "missing prompt for aiQuery");
|
285
292
|
(0, import_utils3.assert)(
|
286
293
|
typeof prompt === "string",
|
@@ -291,6 +298,10 @@ var ScriptPlayer = class {
|
|
291
298
|
} else if ("aiNumber" in flowItem) {
|
292
299
|
const numberTask = flowItem;
|
293
300
|
const prompt = numberTask.aiNumber;
|
301
|
+
const options = {
|
302
|
+
domIncluded: numberTask.domIncluded,
|
303
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
304
|
+
};
|
294
305
|
(0, import_utils3.assert)(prompt, "missing prompt for number");
|
295
306
|
(0, import_utils3.assert)(
|
296
307
|
typeof prompt === "string",
|
@@ -301,6 +312,10 @@ var ScriptPlayer = class {
|
|
301
312
|
} else if ("aiString" in flowItem) {
|
302
313
|
const stringTask = flowItem;
|
303
314
|
const prompt = stringTask.aiString;
|
315
|
+
const options = {
|
316
|
+
domIncluded: stringTask.domIncluded,
|
317
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
318
|
+
};
|
304
319
|
(0, import_utils3.assert)(prompt, "missing prompt for string");
|
305
320
|
(0, import_utils3.assert)(
|
306
321
|
typeof prompt === "string",
|
@@ -311,6 +326,10 @@ var ScriptPlayer = class {
|
|
311
326
|
} else if ("aiBoolean" in flowItem) {
|
312
327
|
const booleanTask = flowItem;
|
313
328
|
const prompt = booleanTask.aiBoolean;
|
329
|
+
const options = {
|
330
|
+
domIncluded: booleanTask.domIncluded,
|
331
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
332
|
+
};
|
314
333
|
(0, import_utils3.assert)(prompt, "missing prompt for boolean");
|
315
334
|
(0, import_utils3.assert)(
|
316
335
|
typeof prompt === "string",
|
@@ -353,6 +372,9 @@ var ScriptPlayer = class {
|
|
353
372
|
} else if ("aiTap" in flowItem) {
|
354
373
|
const tapTask = flowItem;
|
355
374
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
375
|
+
} else if ("aiRightClick" in flowItem) {
|
376
|
+
const rightClickTask = flowItem;
|
377
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
356
378
|
} else if ("aiHover" in flowItem) {
|
357
379
|
const hoverTask = flowItem;
|
358
380
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -375,6 +397,11 @@ var ScriptPlayer = class {
|
|
375
397
|
evaluateJavaScriptTask.javascript
|
376
398
|
);
|
377
399
|
this.setResult(evaluateJavaScriptTask.name, result);
|
400
|
+
} else if ("logScreenshot" in flowItem) {
|
401
|
+
const logScreenshotTask = flowItem;
|
402
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
403
|
+
content: logScreenshotTask.content || ""
|
404
|
+
});
|
378
405
|
} else {
|
379
406
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
380
407
|
}
|
@@ -633,6 +660,94 @@ var replanningCountLimit = 10;
|
|
633
660
|
var isAndroidPage = (page) => {
|
634
661
|
return page.pageType === "android";
|
635
662
|
};
|
663
|
+
var WorkflowMemory = class {
|
664
|
+
constructor(config) {
|
665
|
+
this.workflows = /* @__PURE__ */ new Map();
|
666
|
+
this.config = config;
|
667
|
+
}
|
668
|
+
/**
|
669
|
+
* İş akışı hafızasını getirir
|
670
|
+
*/
|
671
|
+
getWorkflowMemory(workflowId = "default") {
|
672
|
+
const workflow = this.workflows.get(workflowId);
|
673
|
+
return workflow?.memory || [];
|
674
|
+
}
|
675
|
+
/**
|
676
|
+
* İş akışı verilerini getirir
|
677
|
+
*/
|
678
|
+
getWorkflowData(workflowId = "default") {
|
679
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
680
|
+
}
|
681
|
+
/**
|
682
|
+
* İş akışı hafızasını kaydeder
|
683
|
+
*/
|
684
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
685
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
686
|
+
workflow.memory = [...memory];
|
687
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
688
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
689
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
690
|
+
this.workflows.set(workflowId, workflow);
|
691
|
+
this.enforceRetentionPolicy();
|
692
|
+
}
|
693
|
+
/**
|
694
|
+
* İş akışı bağlamını günceller
|
695
|
+
*/
|
696
|
+
updateWorkflowContext(context, workflowId = "default") {
|
697
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
698
|
+
workflow.context = { ...workflow.context, ...context };
|
699
|
+
if (context.currentStep) {
|
700
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
701
|
+
if (!existingStep) {
|
702
|
+
workflow.steps.push({
|
703
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
704
|
+
stepName: context.currentStep,
|
705
|
+
timestamp: context.timestamp,
|
706
|
+
status: "running",
|
707
|
+
memoryItems: []
|
708
|
+
});
|
709
|
+
}
|
710
|
+
}
|
711
|
+
this.workflows.set(workflowId, workflow);
|
712
|
+
}
|
713
|
+
/**
|
714
|
+
* İş akışını temizler
|
715
|
+
*/
|
716
|
+
clearWorkflow(workflowId = "default") {
|
717
|
+
this.workflows.delete(workflowId);
|
718
|
+
}
|
719
|
+
/**
|
720
|
+
* Tüm iş akışlarını temizler
|
721
|
+
*/
|
722
|
+
clearAll() {
|
723
|
+
this.workflows.clear();
|
724
|
+
}
|
725
|
+
createEmptyWorkflowData(workflowId) {
|
726
|
+
return {
|
727
|
+
workflowId,
|
728
|
+
steps: [],
|
729
|
+
memory: [],
|
730
|
+
context: {
|
731
|
+
pageInfo: { url: "", title: "" },
|
732
|
+
timestamp: Date.now()
|
733
|
+
},
|
734
|
+
metadata: {
|
735
|
+
totalSteps: 0,
|
736
|
+
completedSteps: 0,
|
737
|
+
failedSteps: 0,
|
738
|
+
startTime: Date.now()
|
739
|
+
}
|
740
|
+
};
|
741
|
+
}
|
742
|
+
enforceRetentionPolicy() {
|
743
|
+
const maxWorkflows = 10;
|
744
|
+
if (this.workflows.size > maxWorkflows) {
|
745
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
746
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
747
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
748
|
+
}
|
749
|
+
}
|
750
|
+
};
|
636
751
|
var PageTaskExecutor = class {
|
637
752
|
constructor(page, insight, opts) {
|
638
753
|
this.conversationHistory = [];
|
@@ -640,6 +755,25 @@ var PageTaskExecutor = class {
|
|
640
755
|
this.insight = insight;
|
641
756
|
this.taskCache = opts.taskCache;
|
642
757
|
this.onTaskStartCallback = opts?.onTaskStart;
|
758
|
+
this.memoryConfig = {
|
759
|
+
maxItems: 100,
|
760
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
761
|
+
// 2 saat
|
762
|
+
enablePersistence: true,
|
763
|
+
enableAnalytics: true,
|
764
|
+
filterStrategy: "hybrid",
|
765
|
+
...opts?.memoryConfig
|
766
|
+
};
|
767
|
+
this.sessionContext = {
|
768
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
769
|
+
workflowId: opts?.workflowId,
|
770
|
+
startTime: Date.now(),
|
771
|
+
pageInfo: {
|
772
|
+
url: "",
|
773
|
+
title: ""
|
774
|
+
}
|
775
|
+
};
|
776
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
643
777
|
}
|
644
778
|
async recordScreenshot(timing) {
|
645
779
|
const base64 = await this.page.screenshotBase64();
|
@@ -852,8 +986,11 @@ var PageTaskExecutor = class {
|
|
852
986
|
insightDump = dump;
|
853
987
|
};
|
854
988
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
989
|
+
const memoryContext = this.getMemoryAsContext();
|
855
990
|
const assertion = await this.insight.assert(
|
856
|
-
assertPlan.param.assertion
|
991
|
+
assertPlan.param.assertion,
|
992
|
+
memoryContext
|
993
|
+
// Hafıza bağlamını geç
|
857
994
|
);
|
858
995
|
if (!assertion.pass) {
|
859
996
|
if (plan2.type === "Assert") {
|
@@ -890,10 +1027,10 @@ var PageTaskExecutor = class {
|
|
890
1027
|
if (!taskParam || !taskParam.value) {
|
891
1028
|
return;
|
892
1029
|
}
|
893
|
-
await this.page.keyboard.type(taskParam.value);
|
894
|
-
} else {
|
895
|
-
await this.page.keyboard.type(taskParam.value);
|
896
1030
|
}
|
1031
|
+
await this.page.keyboard.type(taskParam.value, {
|
1032
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
1033
|
+
});
|
897
1034
|
}
|
898
1035
|
};
|
899
1036
|
tasks.push(taskActionInput);
|
@@ -922,6 +1059,22 @@ var PageTaskExecutor = class {
|
|
922
1059
|
}
|
923
1060
|
};
|
924
1061
|
tasks.push(taskActionTap);
|
1062
|
+
} else if (plan2.type === "RightClick") {
|
1063
|
+
const taskActionRightClick = {
|
1064
|
+
type: "Action",
|
1065
|
+
subType: "RightClick",
|
1066
|
+
thought: plan2.thought,
|
1067
|
+
locate: plan2.locate,
|
1068
|
+
executor: async (param, { element }) => {
|
1069
|
+
(0, import_utils6.assert)(element, "Element not found, cannot right click");
|
1070
|
+
await this.page.mouse.click(
|
1071
|
+
element.center[0],
|
1072
|
+
element.center[1],
|
1073
|
+
{ button: "right" }
|
1074
|
+
);
|
1075
|
+
}
|
1076
|
+
};
|
1077
|
+
tasks.push(taskActionRightClick);
|
925
1078
|
} else if (plan2.type === "Drag") {
|
926
1079
|
const taskActionDrag = {
|
927
1080
|
type: "Action",
|
@@ -1323,25 +1476,146 @@ var PageTaskExecutor = class {
|
|
1323
1476
|
};
|
1324
1477
|
return task;
|
1325
1478
|
}
|
1479
|
+
/**
|
1480
|
+
* Persistent executor'ı getirir veya oluşturur
|
1481
|
+
*/
|
1482
|
+
getPersistentExecutor() {
|
1483
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1484
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1485
|
+
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1486
|
+
onTaskStart: this.onTaskStartCallback,
|
1487
|
+
initialMemory: previousMemory
|
1488
|
+
});
|
1489
|
+
}
|
1490
|
+
return this.persistentExecutor;
|
1491
|
+
}
|
1492
|
+
/**
|
1493
|
+
* Sayfa bağlamını günceller
|
1494
|
+
*/
|
1495
|
+
async updatePageContext() {
|
1496
|
+
try {
|
1497
|
+
if (this.page.url) {
|
1498
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1499
|
+
}
|
1500
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1501
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1502
|
+
}
|
1503
|
+
} catch (e) {
|
1504
|
+
}
|
1505
|
+
}
|
1506
|
+
/**
|
1507
|
+
* Hafızayı temizler
|
1508
|
+
*/
|
1509
|
+
clearMemory() {
|
1510
|
+
if (this.persistentExecutor) {
|
1511
|
+
this.persistentExecutor.clearMemory();
|
1512
|
+
}
|
1513
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1514
|
+
}
|
1515
|
+
/**
|
1516
|
+
* Mevcut hafızayı döndürür
|
1517
|
+
*/
|
1518
|
+
getMemory() {
|
1519
|
+
return this.persistentExecutor?.getMemory() || [];
|
1520
|
+
}
|
1521
|
+
/**
|
1522
|
+
* İş akışı hafızasını döndürür
|
1523
|
+
*/
|
1524
|
+
getWorkflowMemory() {
|
1525
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1526
|
+
}
|
1527
|
+
/**
|
1528
|
+
* Hafıza istatistiklerini döndürür
|
1529
|
+
*/
|
1530
|
+
getMemoryStats() {
|
1531
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1532
|
+
totalItems: 0,
|
1533
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1534
|
+
config: this.memoryConfig
|
1535
|
+
};
|
1536
|
+
}
|
1537
|
+
/**
|
1538
|
+
* Hafıza konfigürasyonunu günceller
|
1539
|
+
*/
|
1540
|
+
updateMemoryConfig(config) {
|
1541
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1542
|
+
if (this.persistentExecutor) {
|
1543
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1544
|
+
}
|
1545
|
+
}
|
1546
|
+
/**
|
1547
|
+
* Oturum ID'sini oluşturur
|
1548
|
+
*/
|
1549
|
+
generateSessionId() {
|
1550
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1551
|
+
}
|
1552
|
+
/**
|
1553
|
+
* Hafızayı bağlam olarak formatlar
|
1554
|
+
*/
|
1555
|
+
getMemoryAsContext() {
|
1556
|
+
const memory = this.getMemory();
|
1557
|
+
if (memory.length === 0) {
|
1558
|
+
return "";
|
1559
|
+
}
|
1560
|
+
const recentMemory = memory.slice(-5);
|
1561
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1562
|
+
}
|
1326
1563
|
async runPlans(title, plans, opts) {
|
1327
|
-
|
1328
|
-
|
1329
|
-
|
1564
|
+
await this.updatePageContext();
|
1565
|
+
const useMemory = opts?.useMemory !== false;
|
1566
|
+
let taskExecutor;
|
1567
|
+
if (useMemory) {
|
1568
|
+
taskExecutor = this.getPersistentExecutor();
|
1569
|
+
this.workflowMemory.updateWorkflowContext({
|
1570
|
+
currentStep: title,
|
1571
|
+
pageInfo: this.sessionContext.pageInfo,
|
1572
|
+
timestamp: Date.now()
|
1573
|
+
}, this.sessionContext.workflowId || "default");
|
1574
|
+
} else {
|
1575
|
+
taskExecutor = new import_misoai_core.Executor(title, {
|
1576
|
+
onTaskStart: this.onTaskStartCallback
|
1577
|
+
});
|
1578
|
+
}
|
1330
1579
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1580
|
+
tasks.forEach((task) => {
|
1581
|
+
task.context = {
|
1582
|
+
...task.context,
|
1583
|
+
...this.sessionContext.pageInfo,
|
1584
|
+
workflowId: this.sessionContext.workflowId,
|
1585
|
+
sessionId: this.sessionContext.sessionId
|
1586
|
+
};
|
1587
|
+
});
|
1331
1588
|
await taskExecutor.append(tasks);
|
1332
1589
|
const result = await taskExecutor.flush();
|
1590
|
+
if (useMemory) {
|
1591
|
+
this.workflowMemory.saveWorkflowMemory(
|
1592
|
+
taskExecutor.getMemory(),
|
1593
|
+
this.sessionContext.workflowId || "default"
|
1594
|
+
);
|
1595
|
+
}
|
1333
1596
|
return {
|
1334
1597
|
output: result,
|
1335
1598
|
executor: taskExecutor
|
1336
1599
|
};
|
1337
1600
|
}
|
1338
1601
|
async action(userPrompt, actionContext, opts) {
|
1339
|
-
const
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1602
|
+
const useMemory = true;
|
1603
|
+
let taskExecutor;
|
1604
|
+
if (useMemory) {
|
1605
|
+
taskExecutor = this.getPersistentExecutor();
|
1606
|
+
} else {
|
1607
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1608
|
+
onTaskStart: this.onTaskStartCallback
|
1609
|
+
});
|
1610
|
+
}
|
1611
|
+
const memoryContext = this.getMemoryAsContext();
|
1612
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1613
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1343
1614
|
let replanCount = 0;
|
1344
1615
|
const logList = [];
|
1616
|
+
if (memoryContext) {
|
1617
|
+
logList.push(memoryContext);
|
1618
|
+
}
|
1345
1619
|
const yamlFlow = [];
|
1346
1620
|
while (planningTask) {
|
1347
1621
|
if (replanCount > replanningCountLimit) {
|
@@ -1450,16 +1724,22 @@ var PageTaskExecutor = class {
|
|
1450
1724
|
executor: taskExecutor
|
1451
1725
|
};
|
1452
1726
|
}
|
1453
|
-
async createTypeQueryTask(type, demand) {
|
1454
|
-
const
|
1455
|
-
|
1456
|
-
|
1457
|
-
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1727
|
+
async createTypeQueryTask(type, demand, opt) {
|
1728
|
+
const useMemory = true;
|
1729
|
+
let taskExecutor;
|
1730
|
+
if (useMemory) {
|
1731
|
+
taskExecutor = this.getPersistentExecutor();
|
1732
|
+
} else {
|
1733
|
+
taskExecutor = new import_misoai_core.Executor(
|
1734
|
+
taskTitleStr(
|
1735
|
+
type,
|
1736
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1737
|
+
),
|
1738
|
+
{
|
1739
|
+
onTaskStart: this.onTaskStartCallback
|
1740
|
+
}
|
1741
|
+
);
|
1742
|
+
}
|
1463
1743
|
const queryTask = {
|
1464
1744
|
type: "Insight",
|
1465
1745
|
subType: type,
|
@@ -1481,7 +1761,13 @@ var PageTaskExecutor = class {
|
|
1481
1761
|
result: `${type}, ${demand}`
|
1482
1762
|
};
|
1483
1763
|
}
|
1484
|
-
const
|
1764
|
+
const memoryContext = this.getMemoryAsContext();
|
1765
|
+
const { data, usage } = await this.insight.extract(
|
1766
|
+
demandInput,
|
1767
|
+
opt,
|
1768
|
+
memoryContext
|
1769
|
+
// Hafıza bağlamını geç
|
1770
|
+
);
|
1485
1771
|
let outputResult = data;
|
1486
1772
|
if (ifTypeRestricted) {
|
1487
1773
|
(0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
|
@@ -1501,23 +1787,29 @@ var PageTaskExecutor = class {
|
|
1501
1787
|
executor: taskExecutor
|
1502
1788
|
};
|
1503
1789
|
}
|
1504
|
-
async query(demand) {
|
1505
|
-
return this.createTypeQueryTask("Query", demand);
|
1790
|
+
async query(demand, opt) {
|
1791
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1506
1792
|
}
|
1507
|
-
async boolean(prompt) {
|
1508
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1793
|
+
async boolean(prompt, opt) {
|
1794
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1509
1795
|
}
|
1510
|
-
async number(prompt) {
|
1511
|
-
return this.createTypeQueryTask("Number", prompt);
|
1796
|
+
async number(prompt, opt) {
|
1797
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1512
1798
|
}
|
1513
|
-
async string(prompt) {
|
1514
|
-
return this.createTypeQueryTask("String", prompt);
|
1799
|
+
async string(prompt, opt) {
|
1800
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1515
1801
|
}
|
1516
|
-
async assert(assertion) {
|
1802
|
+
async assert(assertion, memoryContext) {
|
1517
1803
|
const description = `assert: ${assertion}`;
|
1518
|
-
const
|
1519
|
-
|
1520
|
-
|
1804
|
+
const useMemory = true;
|
1805
|
+
let taskExecutor;
|
1806
|
+
if (useMemory) {
|
1807
|
+
taskExecutor = this.getPersistentExecutor();
|
1808
|
+
} else {
|
1809
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
|
1810
|
+
onTaskStart: this.onTaskStartCallback
|
1811
|
+
});
|
1812
|
+
}
|
1521
1813
|
const assertionPlan = {
|
1522
1814
|
type: "Assert",
|
1523
1815
|
param: {
|
@@ -1647,7 +1939,7 @@ function buildPlans(type, locateParam, param) {
|
|
1647
1939
|
param: locateParam,
|
1648
1940
|
thought: ""
|
1649
1941
|
} : null;
|
1650
|
-
if (type === "Tap" || type === "Hover") {
|
1942
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1651
1943
|
(0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
|
1652
1944
|
(0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
|
1653
1945
|
const tapPlan = {
|
@@ -1727,7 +2019,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1727
2019
|
var import_semver = __toESM(require("semver"));
|
1728
2020
|
|
1729
2021
|
// package.json
|
1730
|
-
var version = "1.5.
|
2022
|
+
var version = "1.5.7";
|
1731
2023
|
|
1732
2024
|
// src/common/task-cache.ts
|
1733
2025
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1858,8 +2150,14 @@ cache file: ${cacheFile}`
|
|
1858
2150
|
return;
|
1859
2151
|
}
|
1860
2152
|
try {
|
2153
|
+
const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
|
2154
|
+
if (!(0, import_node_fs2.existsSync)(dir)) {
|
2155
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
2156
|
+
debug3("created cache directory: %s", dir);
|
2157
|
+
}
|
1861
2158
|
const yamlData = import_js_yaml3.default.dump(this.cache);
|
1862
2159
|
(0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
|
2160
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1863
2161
|
} catch (err) {
|
1864
2162
|
debug3(
|
1865
2163
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2117,6 +2415,23 @@ var PageAgent = class {
|
|
2117
2415
|
metadata
|
2118
2416
|
};
|
2119
2417
|
}
|
2418
|
+
async aiRightClick(locatePrompt, opt) {
|
2419
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2420
|
+
locatePrompt,
|
2421
|
+
opt
|
2422
|
+
);
|
2423
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2424
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2425
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2426
|
+
plans,
|
2427
|
+
{ cacheable: opt?.cacheable }
|
2428
|
+
);
|
2429
|
+
const metadata = this.afterTaskRunning(executor);
|
2430
|
+
return {
|
2431
|
+
result: output,
|
2432
|
+
metadata
|
2433
|
+
};
|
2434
|
+
}
|
2120
2435
|
async aiInput(value, locatePrompt, opt) {
|
2121
2436
|
(0, import_utils12.assert)(
|
2122
2437
|
typeof value === "string",
|
@@ -2191,7 +2506,13 @@ var PageAgent = class {
|
|
2191
2506
|
metadata: metadata2
|
2192
2507
|
};
|
2193
2508
|
}
|
2194
|
-
const
|
2509
|
+
const memoryContext = this.getMemoryAsContext();
|
2510
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2511
|
+
|
2512
|
+
Previous workflow steps:
|
2513
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2514
|
+
${memoryContext}` : void 0;
|
2515
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2195
2516
|
cacheable
|
2196
2517
|
}));
|
2197
2518
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2337,8 +2658,9 @@ var PageAgent = class {
|
|
2337
2658
|
} catch (e) {
|
2338
2659
|
}
|
2339
2660
|
}
|
2661
|
+
const memoryContext = this.getMemoryAsContext();
|
2340
2662
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2341
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2663
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2342
2664
|
const metadata = this.afterTaskRunning(executor, true);
|
2343
2665
|
if (output && opt?.keepRawResponse) {
|
2344
2666
|
return {
|
@@ -2548,9 +2870,72 @@ ${errors}`);
|
|
2548
2870
|
}
|
2549
2871
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2550
2872
|
}
|
2873
|
+
async logScreenshot(title, options) {
|
2874
|
+
const screenshotTitle = title || "untitled";
|
2875
|
+
const content = options?.content || "";
|
2876
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2877
|
+
if (screenshot) {
|
2878
|
+
const executionDump = {
|
2879
|
+
name: screenshotTitle,
|
2880
|
+
description: content,
|
2881
|
+
tasks: [{
|
2882
|
+
type: "Screenshot",
|
2883
|
+
subType: "log",
|
2884
|
+
status: "finished",
|
2885
|
+
executor: null,
|
2886
|
+
param: {
|
2887
|
+
title: screenshotTitle,
|
2888
|
+
content
|
2889
|
+
},
|
2890
|
+
output: {
|
2891
|
+
screenshot
|
2892
|
+
},
|
2893
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2894
|
+
timing: {
|
2895
|
+
start: Date.now(),
|
2896
|
+
end: Date.now(),
|
2897
|
+
cost: 0
|
2898
|
+
}
|
2899
|
+
}],
|
2900
|
+
sdkVersion: "1.0.0",
|
2901
|
+
logTime: Date.now(),
|
2902
|
+
model_name: "screenshot"
|
2903
|
+
};
|
2904
|
+
this.appendExecutionDump(executionDump);
|
2905
|
+
}
|
2906
|
+
}
|
2551
2907
|
async destroy() {
|
2552
2908
|
await this.page.destroy();
|
2553
2909
|
}
|
2910
|
+
/**
|
2911
|
+
* Hafızayı bağlam olarak formatlar
|
2912
|
+
*/
|
2913
|
+
getMemoryAsContext() {
|
2914
|
+
const memory = this.taskExecutor.getMemory();
|
2915
|
+
if (memory.length === 0) {
|
2916
|
+
return "";
|
2917
|
+
}
|
2918
|
+
const recentMemory = memory.slice(-5);
|
2919
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2920
|
+
}
|
2921
|
+
/**
|
2922
|
+
* Mevcut hafızayı döndürür
|
2923
|
+
*/
|
2924
|
+
getMemory() {
|
2925
|
+
return this.taskExecutor.getMemory();
|
2926
|
+
}
|
2927
|
+
/**
|
2928
|
+
* Hafıza istatistiklerini döndürür
|
2929
|
+
*/
|
2930
|
+
getMemoryStats() {
|
2931
|
+
return this.taskExecutor.getMemoryStats();
|
2932
|
+
}
|
2933
|
+
/**
|
2934
|
+
* Hafızayı temizler
|
2935
|
+
*/
|
2936
|
+
clearMemory() {
|
2937
|
+
this.taskExecutor.clearMemory();
|
2938
|
+
}
|
2554
2939
|
};
|
2555
2940
|
|
2556
2941
|
// src/chrome-extension/agent.ts
|
@@ -2768,7 +3153,7 @@ function sleep2(ms) {
|
|
2768
3153
|
var ChromeExtensionProxyPage = class {
|
2769
3154
|
constructor(forceSameTabNavigation) {
|
2770
3155
|
this.pageType = "chrome-extension-proxy";
|
2771
|
-
this.version = "1.5.
|
3156
|
+
this.version = "1.5.7";
|
2772
3157
|
this.activeTabId = null;
|
2773
3158
|
this.tabIdOfDebuggerAttached = null;
|
2774
3159
|
this.attachingDebugger = null;
|
@@ -2777,7 +3162,8 @@ var ChromeExtensionProxyPage = class {
|
|
2777
3162
|
this.latestMouseX = 100;
|
2778
3163
|
this.latestMouseY = 100;
|
2779
3164
|
this.mouse = {
|
2780
|
-
click: async (x, y) => {
|
3165
|
+
click: async (x, y, options) => {
|
3166
|
+
const { button = "left", count = 1 } = options || {};
|
2781
3167
|
await this.mouse.move(x, y);
|
2782
3168
|
if (this.isMobileEmulation === null) {
|
2783
3169
|
const result = await this.sendCommandToDebugger("Runtime.evaluate", {
|
@@ -2788,7 +3174,7 @@ var ChromeExtensionProxyPage = class {
|
|
2788
3174
|
});
|
2789
3175
|
this.isMobileEmulation = result?.result?.value;
|
2790
3176
|
}
|
2791
|
-
if (this.isMobileEmulation) {
|
3177
|
+
if (this.isMobileEmulation && button === "left") {
|
2792
3178
|
const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
|
2793
3179
|
await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
|
2794
3180
|
type: "touchStart",
|
@@ -2805,15 +3191,15 @@ var ChromeExtensionProxyPage = class {
|
|
2805
3191
|
type: "mousePressed",
|
2806
3192
|
x,
|
2807
3193
|
y,
|
2808
|
-
button
|
2809
|
-
clickCount:
|
3194
|
+
button,
|
3195
|
+
clickCount: count
|
2810
3196
|
});
|
2811
3197
|
await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
|
2812
3198
|
type: "mouseReleased",
|
2813
3199
|
x,
|
2814
3200
|
y,
|
2815
|
-
button
|
2816
|
-
clickCount:
|
3201
|
+
button,
|
3202
|
+
clickCount: count
|
2817
3203
|
});
|
2818
3204
|
}
|
2819
3205
|
},
|