misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
package/dist/es/agent.js
CHANGED
@@ -105,6 +105,10 @@ var ScriptPlayer = class {
|
|
105
105
|
} else if ("aiQuery" in flowItem) {
|
106
106
|
const queryTask = flowItem;
|
107
107
|
const prompt = queryTask.aiQuery;
|
108
|
+
const options = {
|
109
|
+
domIncluded: queryTask.domIncluded,
|
110
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
111
|
+
};
|
108
112
|
assert(prompt, "missing prompt for aiQuery");
|
109
113
|
assert(
|
110
114
|
typeof prompt === "string",
|
@@ -115,6 +119,10 @@ var ScriptPlayer = class {
|
|
115
119
|
} else if ("aiNumber" in flowItem) {
|
116
120
|
const numberTask = flowItem;
|
117
121
|
const prompt = numberTask.aiNumber;
|
122
|
+
const options = {
|
123
|
+
domIncluded: numberTask.domIncluded,
|
124
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
125
|
+
};
|
118
126
|
assert(prompt, "missing prompt for number");
|
119
127
|
assert(
|
120
128
|
typeof prompt === "string",
|
@@ -125,6 +133,10 @@ var ScriptPlayer = class {
|
|
125
133
|
} else if ("aiString" in flowItem) {
|
126
134
|
const stringTask = flowItem;
|
127
135
|
const prompt = stringTask.aiString;
|
136
|
+
const options = {
|
137
|
+
domIncluded: stringTask.domIncluded,
|
138
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
139
|
+
};
|
128
140
|
assert(prompt, "missing prompt for string");
|
129
141
|
assert(
|
130
142
|
typeof prompt === "string",
|
@@ -135,6 +147,10 @@ var ScriptPlayer = class {
|
|
135
147
|
} else if ("aiBoolean" in flowItem) {
|
136
148
|
const booleanTask = flowItem;
|
137
149
|
const prompt = booleanTask.aiBoolean;
|
150
|
+
const options = {
|
151
|
+
domIncluded: booleanTask.domIncluded,
|
152
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
153
|
+
};
|
138
154
|
assert(prompt, "missing prompt for boolean");
|
139
155
|
assert(
|
140
156
|
typeof prompt === "string",
|
@@ -177,6 +193,9 @@ var ScriptPlayer = class {
|
|
177
193
|
} else if ("aiTap" in flowItem) {
|
178
194
|
const tapTask = flowItem;
|
179
195
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
196
|
+
} else if ("aiRightClick" in flowItem) {
|
197
|
+
const rightClickTask = flowItem;
|
198
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
180
199
|
} else if ("aiHover" in flowItem) {
|
181
200
|
const hoverTask = flowItem;
|
182
201
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -199,6 +218,11 @@ var ScriptPlayer = class {
|
|
199
218
|
evaluateJavaScriptTask.javascript
|
200
219
|
);
|
201
220
|
this.setResult(evaluateJavaScriptTask.name, result);
|
221
|
+
} else if ("logScreenshot" in flowItem) {
|
222
|
+
const logScreenshotTask = flowItem;
|
223
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
224
|
+
content: logScreenshotTask.content || ""
|
225
|
+
});
|
202
226
|
} else {
|
203
227
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
204
228
|
}
|
@@ -467,7 +491,8 @@ var WebElementInfo = class {
|
|
467
491
|
id,
|
468
492
|
attributes,
|
469
493
|
indexId,
|
470
|
-
xpaths
|
494
|
+
xpaths,
|
495
|
+
isVisible
|
471
496
|
}) {
|
472
497
|
this.content = content;
|
473
498
|
this.rect = rect;
|
@@ -480,6 +505,7 @@ var WebElementInfo = class {
|
|
480
505
|
this.attributes = attributes;
|
481
506
|
this.indexId = indexId;
|
482
507
|
this.xpaths = xpaths;
|
508
|
+
this.isVisible = isVisible;
|
483
509
|
}
|
484
510
|
};
|
485
511
|
|
@@ -502,14 +528,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
502
528
|
})
|
503
529
|
]);
|
504
530
|
const webTree = traverseTree(tree, (elementInfo) => {
|
505
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
531
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
506
532
|
return new WebElementInfo({
|
507
533
|
rect,
|
508
534
|
locator,
|
509
535
|
id,
|
510
536
|
content,
|
511
537
|
attributes,
|
512
|
-
indexId
|
538
|
+
indexId,
|
539
|
+
isVisible
|
513
540
|
});
|
514
541
|
});
|
515
542
|
assert3(screenshotBase64, "screenshotBase64 is required");
|
@@ -539,7 +566,7 @@ function printReportMsg(filepath) {
|
|
539
566
|
logMsg(`Midscene - report file updated: ${filepath}`);
|
540
567
|
}
|
541
568
|
function replaceIllegalPathCharsAndSpace(str) {
|
542
|
-
return str.replace(/[
|
569
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
543
570
|
}
|
544
571
|
function matchElementFromPlan(planLocateParam, tree) {
|
545
572
|
if (!planLocateParam) {
|
@@ -568,6 +595,94 @@ var replanningCountLimit = 10;
|
|
568
595
|
var isAndroidPage = (page) => {
|
569
596
|
return page.pageType === "android";
|
570
597
|
};
|
598
|
+
var WorkflowMemory = class {
|
599
|
+
constructor(config) {
|
600
|
+
this.workflows = /* @__PURE__ */ new Map();
|
601
|
+
this.config = config;
|
602
|
+
}
|
603
|
+
/**
|
604
|
+
* İş akışı hafızasını getirir
|
605
|
+
*/
|
606
|
+
getWorkflowMemory(workflowId = "default") {
|
607
|
+
const workflow = this.workflows.get(workflowId);
|
608
|
+
return workflow?.memory || [];
|
609
|
+
}
|
610
|
+
/**
|
611
|
+
* İş akışı verilerini getirir
|
612
|
+
*/
|
613
|
+
getWorkflowData(workflowId = "default") {
|
614
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
615
|
+
}
|
616
|
+
/**
|
617
|
+
* İş akışı hafızasını kaydeder
|
618
|
+
*/
|
619
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
620
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
621
|
+
workflow.memory = [...memory];
|
622
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
623
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
624
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
625
|
+
this.workflows.set(workflowId, workflow);
|
626
|
+
this.enforceRetentionPolicy();
|
627
|
+
}
|
628
|
+
/**
|
629
|
+
* İş akışı bağlamını günceller
|
630
|
+
*/
|
631
|
+
updateWorkflowContext(context, workflowId = "default") {
|
632
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
633
|
+
workflow.context = { ...workflow.context, ...context };
|
634
|
+
if (context.currentStep) {
|
635
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
636
|
+
if (!existingStep) {
|
637
|
+
workflow.steps.push({
|
638
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
639
|
+
stepName: context.currentStep,
|
640
|
+
timestamp: context.timestamp,
|
641
|
+
status: "running",
|
642
|
+
memoryItems: []
|
643
|
+
});
|
644
|
+
}
|
645
|
+
}
|
646
|
+
this.workflows.set(workflowId, workflow);
|
647
|
+
}
|
648
|
+
/**
|
649
|
+
* İş akışını temizler
|
650
|
+
*/
|
651
|
+
clearWorkflow(workflowId = "default") {
|
652
|
+
this.workflows.delete(workflowId);
|
653
|
+
}
|
654
|
+
/**
|
655
|
+
* Tüm iş akışlarını temizler
|
656
|
+
*/
|
657
|
+
clearAll() {
|
658
|
+
this.workflows.clear();
|
659
|
+
}
|
660
|
+
createEmptyWorkflowData(workflowId) {
|
661
|
+
return {
|
662
|
+
workflowId,
|
663
|
+
steps: [],
|
664
|
+
memory: [],
|
665
|
+
context: {
|
666
|
+
pageInfo: { url: "", title: "" },
|
667
|
+
timestamp: Date.now()
|
668
|
+
},
|
669
|
+
metadata: {
|
670
|
+
totalSteps: 0,
|
671
|
+
completedSteps: 0,
|
672
|
+
failedSteps: 0,
|
673
|
+
startTime: Date.now()
|
674
|
+
}
|
675
|
+
};
|
676
|
+
}
|
677
|
+
enforceRetentionPolicy() {
|
678
|
+
const maxWorkflows = 10;
|
679
|
+
if (this.workflows.size > maxWorkflows) {
|
680
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
681
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
682
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
683
|
+
}
|
684
|
+
}
|
685
|
+
};
|
571
686
|
var PageTaskExecutor = class {
|
572
687
|
constructor(page, insight, opts) {
|
573
688
|
this.conversationHistory = [];
|
@@ -575,6 +690,25 @@ var PageTaskExecutor = class {
|
|
575
690
|
this.insight = insight;
|
576
691
|
this.taskCache = opts.taskCache;
|
577
692
|
this.onTaskStartCallback = opts?.onTaskStart;
|
693
|
+
this.memoryConfig = {
|
694
|
+
maxItems: 100,
|
695
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
696
|
+
// 2 saat
|
697
|
+
enablePersistence: true,
|
698
|
+
enableAnalytics: true,
|
699
|
+
filterStrategy: "hybrid",
|
700
|
+
...opts?.memoryConfig
|
701
|
+
};
|
702
|
+
this.sessionContext = {
|
703
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
704
|
+
workflowId: opts?.workflowId,
|
705
|
+
startTime: Date.now(),
|
706
|
+
pageInfo: {
|
707
|
+
url: "",
|
708
|
+
title: ""
|
709
|
+
}
|
710
|
+
};
|
711
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
578
712
|
}
|
579
713
|
async recordScreenshot(timing) {
|
580
714
|
const base64 = await this.page.screenshotBase64();
|
@@ -787,8 +921,11 @@ var PageTaskExecutor = class {
|
|
787
921
|
insightDump = dump;
|
788
922
|
};
|
789
923
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
924
|
+
const memoryContext = this.getMemoryAsContext();
|
790
925
|
const assertion = await this.insight.assert(
|
791
|
-
assertPlan.param.assertion
|
926
|
+
assertPlan.param.assertion,
|
927
|
+
memoryContext
|
928
|
+
// Hafıza bağlamını geç
|
792
929
|
);
|
793
930
|
if (!assertion.pass) {
|
794
931
|
if (plan2.type === "Assert") {
|
@@ -825,10 +962,10 @@ var PageTaskExecutor = class {
|
|
825
962
|
if (!taskParam || !taskParam.value) {
|
826
963
|
return;
|
827
964
|
}
|
828
|
-
await this.page.keyboard.type(taskParam.value);
|
829
|
-
} else {
|
830
|
-
await this.page.keyboard.type(taskParam.value);
|
831
965
|
}
|
966
|
+
await this.page.keyboard.type(taskParam.value, {
|
967
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
968
|
+
});
|
832
969
|
}
|
833
970
|
};
|
834
971
|
tasks.push(taskActionInput);
|
@@ -857,6 +994,22 @@ var PageTaskExecutor = class {
|
|
857
994
|
}
|
858
995
|
};
|
859
996
|
tasks.push(taskActionTap);
|
997
|
+
} else if (plan2.type === "RightClick") {
|
998
|
+
const taskActionRightClick = {
|
999
|
+
type: "Action",
|
1000
|
+
subType: "RightClick",
|
1001
|
+
thought: plan2.thought,
|
1002
|
+
locate: plan2.locate,
|
1003
|
+
executor: async (param, { element }) => {
|
1004
|
+
assert4(element, "Element not found, cannot right click");
|
1005
|
+
await this.page.mouse.click(
|
1006
|
+
element.center[0],
|
1007
|
+
element.center[1],
|
1008
|
+
{ button: "right" }
|
1009
|
+
);
|
1010
|
+
}
|
1011
|
+
};
|
1012
|
+
tasks.push(taskActionRightClick);
|
860
1013
|
} else if (plan2.type === "Drag") {
|
861
1014
|
const taskActionDrag = {
|
862
1015
|
type: "Action",
|
@@ -1258,25 +1411,146 @@ var PageTaskExecutor = class {
|
|
1258
1411
|
};
|
1259
1412
|
return task;
|
1260
1413
|
}
|
1414
|
+
/**
|
1415
|
+
* Persistent executor'ı getirir veya oluşturur
|
1416
|
+
*/
|
1417
|
+
getPersistentExecutor() {
|
1418
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1419
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1420
|
+
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1421
|
+
onTaskStart: this.onTaskStartCallback,
|
1422
|
+
initialMemory: previousMemory
|
1423
|
+
});
|
1424
|
+
}
|
1425
|
+
return this.persistentExecutor;
|
1426
|
+
}
|
1427
|
+
/**
|
1428
|
+
* Sayfa bağlamını günceller
|
1429
|
+
*/
|
1430
|
+
async updatePageContext() {
|
1431
|
+
try {
|
1432
|
+
if (this.page.url) {
|
1433
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1434
|
+
}
|
1435
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1436
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1437
|
+
}
|
1438
|
+
} catch (e) {
|
1439
|
+
}
|
1440
|
+
}
|
1441
|
+
/**
|
1442
|
+
* Hafızayı temizler
|
1443
|
+
*/
|
1444
|
+
clearMemory() {
|
1445
|
+
if (this.persistentExecutor) {
|
1446
|
+
this.persistentExecutor.clearMemory();
|
1447
|
+
}
|
1448
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1449
|
+
}
|
1450
|
+
/**
|
1451
|
+
* Mevcut hafızayı döndürür
|
1452
|
+
*/
|
1453
|
+
getMemory() {
|
1454
|
+
return this.persistentExecutor?.getMemory() || [];
|
1455
|
+
}
|
1456
|
+
/**
|
1457
|
+
* İş akışı hafızasını döndürür
|
1458
|
+
*/
|
1459
|
+
getWorkflowMemory() {
|
1460
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1461
|
+
}
|
1462
|
+
/**
|
1463
|
+
* Hafıza istatistiklerini döndürür
|
1464
|
+
*/
|
1465
|
+
getMemoryStats() {
|
1466
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1467
|
+
totalItems: 0,
|
1468
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1469
|
+
config: this.memoryConfig
|
1470
|
+
};
|
1471
|
+
}
|
1472
|
+
/**
|
1473
|
+
* Hafıza konfigürasyonunu günceller
|
1474
|
+
*/
|
1475
|
+
updateMemoryConfig(config) {
|
1476
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1477
|
+
if (this.persistentExecutor) {
|
1478
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1479
|
+
}
|
1480
|
+
}
|
1481
|
+
/**
|
1482
|
+
* Oturum ID'sini oluşturur
|
1483
|
+
*/
|
1484
|
+
generateSessionId() {
|
1485
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1486
|
+
}
|
1487
|
+
/**
|
1488
|
+
* Hafızayı bağlam olarak formatlar
|
1489
|
+
*/
|
1490
|
+
getMemoryAsContext() {
|
1491
|
+
const memory = this.getMemory();
|
1492
|
+
if (memory.length === 0) {
|
1493
|
+
return "";
|
1494
|
+
}
|
1495
|
+
const recentMemory = memory.slice(-5);
|
1496
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1497
|
+
}
|
1261
1498
|
async runPlans(title, plans, opts) {
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1499
|
+
await this.updatePageContext();
|
1500
|
+
const useMemory = opts?.useMemory !== false;
|
1501
|
+
let taskExecutor;
|
1502
|
+
if (useMemory) {
|
1503
|
+
taskExecutor = this.getPersistentExecutor();
|
1504
|
+
this.workflowMemory.updateWorkflowContext({
|
1505
|
+
currentStep: title,
|
1506
|
+
pageInfo: this.sessionContext.pageInfo,
|
1507
|
+
timestamp: Date.now()
|
1508
|
+
}, this.sessionContext.workflowId || "default");
|
1509
|
+
} else {
|
1510
|
+
taskExecutor = new Executor(title, {
|
1511
|
+
onTaskStart: this.onTaskStartCallback
|
1512
|
+
});
|
1513
|
+
}
|
1265
1514
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1515
|
+
tasks.forEach((task) => {
|
1516
|
+
task.context = {
|
1517
|
+
...task.context,
|
1518
|
+
...this.sessionContext.pageInfo,
|
1519
|
+
workflowId: this.sessionContext.workflowId,
|
1520
|
+
sessionId: this.sessionContext.sessionId
|
1521
|
+
};
|
1522
|
+
});
|
1266
1523
|
await taskExecutor.append(tasks);
|
1267
1524
|
const result = await taskExecutor.flush();
|
1525
|
+
if (useMemory) {
|
1526
|
+
this.workflowMemory.saveWorkflowMemory(
|
1527
|
+
taskExecutor.getMemory(),
|
1528
|
+
this.sessionContext.workflowId || "default"
|
1529
|
+
);
|
1530
|
+
}
|
1268
1531
|
return {
|
1269
1532
|
output: result,
|
1270
1533
|
executor: taskExecutor
|
1271
1534
|
};
|
1272
1535
|
}
|
1273
1536
|
async action(userPrompt, actionContext, opts) {
|
1274
|
-
const
|
1275
|
-
|
1276
|
-
|
1277
|
-
|
1537
|
+
const useMemory = true;
|
1538
|
+
let taskExecutor;
|
1539
|
+
if (useMemory) {
|
1540
|
+
taskExecutor = this.getPersistentExecutor();
|
1541
|
+
} else {
|
1542
|
+
taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
|
1543
|
+
onTaskStart: this.onTaskStartCallback
|
1544
|
+
});
|
1545
|
+
}
|
1546
|
+
const memoryContext = this.getMemoryAsContext();
|
1547
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1548
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1278
1549
|
let replanCount = 0;
|
1279
1550
|
const logList = [];
|
1551
|
+
if (memoryContext) {
|
1552
|
+
logList.push(memoryContext);
|
1553
|
+
}
|
1280
1554
|
const yamlFlow = [];
|
1281
1555
|
while (planningTask) {
|
1282
1556
|
if (replanCount > replanningCountLimit) {
|
@@ -1385,16 +1659,22 @@ var PageTaskExecutor = class {
|
|
1385
1659
|
executor: taskExecutor
|
1386
1660
|
};
|
1387
1661
|
}
|
1388
|
-
async createTypeQueryTask(type, demand) {
|
1389
|
-
const
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
|
1394
|
-
|
1395
|
-
|
1396
|
-
|
1397
|
-
|
1662
|
+
async createTypeQueryTask(type, demand, opt) {
|
1663
|
+
const useMemory = true;
|
1664
|
+
let taskExecutor;
|
1665
|
+
if (useMemory) {
|
1666
|
+
taskExecutor = this.getPersistentExecutor();
|
1667
|
+
} else {
|
1668
|
+
taskExecutor = new Executor(
|
1669
|
+
taskTitleStr(
|
1670
|
+
type,
|
1671
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1672
|
+
),
|
1673
|
+
{
|
1674
|
+
onTaskStart: this.onTaskStartCallback
|
1675
|
+
}
|
1676
|
+
);
|
1677
|
+
}
|
1398
1678
|
const queryTask = {
|
1399
1679
|
type: "Insight",
|
1400
1680
|
subType: type,
|
@@ -1416,7 +1696,13 @@ var PageTaskExecutor = class {
|
|
1416
1696
|
result: `${type}, ${demand}`
|
1417
1697
|
};
|
1418
1698
|
}
|
1419
|
-
const
|
1699
|
+
const memoryContext = this.getMemoryAsContext();
|
1700
|
+
const { data, usage } = await this.insight.extract(
|
1701
|
+
demandInput,
|
1702
|
+
opt,
|
1703
|
+
memoryContext
|
1704
|
+
// Hafıza bağlamını geç
|
1705
|
+
);
|
1420
1706
|
let outputResult = data;
|
1421
1707
|
if (ifTypeRestricted) {
|
1422
1708
|
assert4(data?.result !== void 0, "No result in query data");
|
@@ -1436,23 +1722,29 @@ var PageTaskExecutor = class {
|
|
1436
1722
|
executor: taskExecutor
|
1437
1723
|
};
|
1438
1724
|
}
|
1439
|
-
async query(demand) {
|
1440
|
-
return this.createTypeQueryTask("Query", demand);
|
1725
|
+
async query(demand, opt) {
|
1726
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1441
1727
|
}
|
1442
|
-
async boolean(prompt) {
|
1443
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1728
|
+
async boolean(prompt, opt) {
|
1729
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1444
1730
|
}
|
1445
|
-
async number(prompt) {
|
1446
|
-
return this.createTypeQueryTask("Number", prompt);
|
1731
|
+
async number(prompt, opt) {
|
1732
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1447
1733
|
}
|
1448
|
-
async string(prompt) {
|
1449
|
-
return this.createTypeQueryTask("String", prompt);
|
1734
|
+
async string(prompt, opt) {
|
1735
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1450
1736
|
}
|
1451
|
-
async assert(assertion) {
|
1737
|
+
async assert(assertion, memoryContext) {
|
1452
1738
|
const description = `assert: ${assertion}`;
|
1453
|
-
const
|
1454
|
-
|
1455
|
-
|
1739
|
+
const useMemory = true;
|
1740
|
+
let taskExecutor;
|
1741
|
+
if (useMemory) {
|
1742
|
+
taskExecutor = this.getPersistentExecutor();
|
1743
|
+
} else {
|
1744
|
+
taskExecutor = new Executor(taskTitleStr("Assert", description), {
|
1745
|
+
onTaskStart: this.onTaskStartCallback
|
1746
|
+
});
|
1747
|
+
}
|
1456
1748
|
const assertionPlan = {
|
1457
1749
|
type: "Assert",
|
1458
1750
|
param: {
|
@@ -1582,7 +1874,7 @@ function buildPlans(type, locateParam, param) {
|
|
1582
1874
|
param: locateParam,
|
1583
1875
|
thought: ""
|
1584
1876
|
} : null;
|
1585
|
-
if (type === "Tap" || type === "Hover") {
|
1877
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1586
1878
|
assert5(locateParam, `missing locate info for action "${type}"`);
|
1587
1879
|
assert5(locatePlan, `missing locate info for action "${type}"`);
|
1588
1880
|
const tapPlan = {
|
@@ -1653,8 +1945,8 @@ function buildPlans(type, locateParam, param) {
|
|
1653
1945
|
|
1654
1946
|
// src/common/task-cache.ts
|
1655
1947
|
import assert6 from "assert";
|
1656
|
-
import { existsSync as existsSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
1657
|
-
import { join as join2 } from "path";
|
1948
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
1949
|
+
import { dirname as dirname2, join as join2 } from "path";
|
1658
1950
|
import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
|
1659
1951
|
import { getDebug as getDebug3 } from "misoai-shared/logger";
|
1660
1952
|
import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
|
@@ -1662,7 +1954,7 @@ import yaml3 from "js-yaml";
|
|
1662
1954
|
import semver from "semver";
|
1663
1955
|
|
1664
1956
|
// package.json
|
1665
|
-
var version = "1.5.
|
1957
|
+
var version = "1.5.7";
|
1666
1958
|
|
1667
1959
|
// src/common/task-cache.ts
|
1668
1960
|
var debug3 = getDebug3("cache");
|
@@ -1793,8 +2085,14 @@ cache file: ${cacheFile}`
|
|
1793
2085
|
return;
|
1794
2086
|
}
|
1795
2087
|
try {
|
2088
|
+
const dir = dirname2(this.cacheFilePath);
|
2089
|
+
if (!existsSync2(dir)) {
|
2090
|
+
mkdirSync2(dir, { recursive: true });
|
2091
|
+
debug3("created cache directory: %s", dir);
|
2092
|
+
}
|
1796
2093
|
const yamlData = yaml3.dump(this.cache);
|
1797
2094
|
writeFileSync2(this.cacheFilePath, yamlData);
|
2095
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1798
2096
|
} catch (err) {
|
1799
2097
|
debug3(
|
1800
2098
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2052,6 +2350,23 @@ var PageAgent = class {
|
|
2052
2350
|
metadata
|
2053
2351
|
};
|
2054
2352
|
}
|
2353
|
+
async aiRightClick(locatePrompt, opt) {
|
2354
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2355
|
+
locatePrompt,
|
2356
|
+
opt
|
2357
|
+
);
|
2358
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2359
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2360
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2361
|
+
plans,
|
2362
|
+
{ cacheable: opt?.cacheable }
|
2363
|
+
);
|
2364
|
+
const metadata = this.afterTaskRunning(executor);
|
2365
|
+
return {
|
2366
|
+
result: output,
|
2367
|
+
metadata
|
2368
|
+
};
|
2369
|
+
}
|
2055
2370
|
async aiInput(value, locatePrompt, opt) {
|
2056
2371
|
assert7(
|
2057
2372
|
typeof value === "string",
|
@@ -2126,7 +2441,13 @@ var PageAgent = class {
|
|
2126
2441
|
metadata: metadata2
|
2127
2442
|
};
|
2128
2443
|
}
|
2129
|
-
const
|
2444
|
+
const memoryContext = this.getMemoryAsContext();
|
2445
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2446
|
+
|
2447
|
+
Previous workflow steps:
|
2448
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2449
|
+
${memoryContext}` : void 0;
|
2450
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2130
2451
|
cacheable
|
2131
2452
|
}));
|
2132
2453
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2272,8 +2593,9 @@ var PageAgent = class {
|
|
2272
2593
|
} catch (e) {
|
2273
2594
|
}
|
2274
2595
|
}
|
2596
|
+
const memoryContext = this.getMemoryAsContext();
|
2275
2597
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2276
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2598
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2277
2599
|
const metadata = this.afterTaskRunning(executor, true);
|
2278
2600
|
if (output && opt?.keepRawResponse) {
|
2279
2601
|
return {
|
@@ -2483,9 +2805,72 @@ ${errors}`);
|
|
2483
2805
|
}
|
2484
2806
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2485
2807
|
}
|
2808
|
+
async logScreenshot(title, options) {
|
2809
|
+
const screenshotTitle = title || "untitled";
|
2810
|
+
const content = options?.content || "";
|
2811
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2812
|
+
if (screenshot) {
|
2813
|
+
const executionDump = {
|
2814
|
+
name: screenshotTitle,
|
2815
|
+
description: content,
|
2816
|
+
tasks: [{
|
2817
|
+
type: "Screenshot",
|
2818
|
+
subType: "log",
|
2819
|
+
status: "finished",
|
2820
|
+
executor: null,
|
2821
|
+
param: {
|
2822
|
+
title: screenshotTitle,
|
2823
|
+
content
|
2824
|
+
},
|
2825
|
+
output: {
|
2826
|
+
screenshot
|
2827
|
+
},
|
2828
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2829
|
+
timing: {
|
2830
|
+
start: Date.now(),
|
2831
|
+
end: Date.now(),
|
2832
|
+
cost: 0
|
2833
|
+
}
|
2834
|
+
}],
|
2835
|
+
sdkVersion: "1.0.0",
|
2836
|
+
logTime: Date.now(),
|
2837
|
+
model_name: "screenshot"
|
2838
|
+
};
|
2839
|
+
this.appendExecutionDump(executionDump);
|
2840
|
+
}
|
2841
|
+
}
|
2486
2842
|
async destroy() {
|
2487
2843
|
await this.page.destroy();
|
2488
2844
|
}
|
2845
|
+
/**
|
2846
|
+
* Hafızayı bağlam olarak formatlar
|
2847
|
+
*/
|
2848
|
+
getMemoryAsContext() {
|
2849
|
+
const memory = this.taskExecutor.getMemory();
|
2850
|
+
if (memory.length === 0) {
|
2851
|
+
return "";
|
2852
|
+
}
|
2853
|
+
const recentMemory = memory.slice(-5);
|
2854
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2855
|
+
}
|
2856
|
+
/**
|
2857
|
+
* Mevcut hafızayı döndürür
|
2858
|
+
*/
|
2859
|
+
getMemory() {
|
2860
|
+
return this.taskExecutor.getMemory();
|
2861
|
+
}
|
2862
|
+
/**
|
2863
|
+
* Hafıza istatistiklerini döndürür
|
2864
|
+
*/
|
2865
|
+
getMemoryStats() {
|
2866
|
+
return this.taskExecutor.getMemoryStats();
|
2867
|
+
}
|
2868
|
+
/**
|
2869
|
+
* Hafızayı temizler
|
2870
|
+
*/
|
2871
|
+
clearMemory() {
|
2872
|
+
this.taskExecutor.clearMemory();
|
2873
|
+
}
|
2489
2874
|
};
|
2490
2875
|
export {
|
2491
2876
|
PageAgent
|