misoai-web 1.5.6 → 1.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +429 -44
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +431 -46
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +437 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +445 -44
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +429 -44
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +429 -44
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +445 -44
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +429 -44
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +429 -44
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +24 -0
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +427 -42
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +429 -44
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +435 -49
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +443 -42
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +427 -42
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +427 -42
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +443 -42
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +427 -42
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +427 -42
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +24 -0
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +101 -8
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +51 -73
- package/iife-script/htmlElementDebug.js +33 -54
- package/package.json +23 -23
- package/LICENSE +0 -21
package/dist/lib/bridge-mode.js
CHANGED
@@ -144,6 +144,10 @@ var ScriptPlayer = class {
|
|
144
144
|
} else if ("aiQuery" in flowItem) {
|
145
145
|
const queryTask = flowItem;
|
146
146
|
const prompt = queryTask.aiQuery;
|
147
|
+
const options = {
|
148
|
+
domIncluded: queryTask.domIncluded,
|
149
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
150
|
+
};
|
147
151
|
(0, import_utils.assert)(prompt, "missing prompt for aiQuery");
|
148
152
|
(0, import_utils.assert)(
|
149
153
|
typeof prompt === "string",
|
@@ -154,6 +158,10 @@ var ScriptPlayer = class {
|
|
154
158
|
} else if ("aiNumber" in flowItem) {
|
155
159
|
const numberTask = flowItem;
|
156
160
|
const prompt = numberTask.aiNumber;
|
161
|
+
const options = {
|
162
|
+
domIncluded: numberTask.domIncluded,
|
163
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
164
|
+
};
|
157
165
|
(0, import_utils.assert)(prompt, "missing prompt for number");
|
158
166
|
(0, import_utils.assert)(
|
159
167
|
typeof prompt === "string",
|
@@ -164,6 +172,10 @@ var ScriptPlayer = class {
|
|
164
172
|
} else if ("aiString" in flowItem) {
|
165
173
|
const stringTask = flowItem;
|
166
174
|
const prompt = stringTask.aiString;
|
175
|
+
const options = {
|
176
|
+
domIncluded: stringTask.domIncluded,
|
177
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
178
|
+
};
|
167
179
|
(0, import_utils.assert)(prompt, "missing prompt for string");
|
168
180
|
(0, import_utils.assert)(
|
169
181
|
typeof prompt === "string",
|
@@ -174,6 +186,10 @@ var ScriptPlayer = class {
|
|
174
186
|
} else if ("aiBoolean" in flowItem) {
|
175
187
|
const booleanTask = flowItem;
|
176
188
|
const prompt = booleanTask.aiBoolean;
|
189
|
+
const options = {
|
190
|
+
domIncluded: booleanTask.domIncluded,
|
191
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
192
|
+
};
|
177
193
|
(0, import_utils.assert)(prompt, "missing prompt for boolean");
|
178
194
|
(0, import_utils.assert)(
|
179
195
|
typeof prompt === "string",
|
@@ -216,6 +232,9 @@ var ScriptPlayer = class {
|
|
216
232
|
} else if ("aiTap" in flowItem) {
|
217
233
|
const tapTask = flowItem;
|
218
234
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
235
|
+
} else if ("aiRightClick" in flowItem) {
|
236
|
+
const rightClickTask = flowItem;
|
237
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
219
238
|
} else if ("aiHover" in flowItem) {
|
220
239
|
const hoverTask = flowItem;
|
221
240
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -238,6 +257,11 @@ var ScriptPlayer = class {
|
|
238
257
|
evaluateJavaScriptTask.javascript
|
239
258
|
);
|
240
259
|
this.setResult(evaluateJavaScriptTask.name, result);
|
260
|
+
} else if ("logScreenshot" in flowItem) {
|
261
|
+
const logScreenshotTask = flowItem;
|
262
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
263
|
+
content: logScreenshotTask.content || ""
|
264
|
+
});
|
241
265
|
} else {
|
242
266
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
243
267
|
}
|
@@ -486,7 +510,8 @@ var WebElementInfo = class {
|
|
486
510
|
id,
|
487
511
|
attributes,
|
488
512
|
indexId,
|
489
|
-
xpaths
|
513
|
+
xpaths,
|
514
|
+
isVisible
|
490
515
|
}) {
|
491
516
|
this.content = content;
|
492
517
|
this.rect = rect;
|
@@ -499,6 +524,7 @@ var WebElementInfo = class {
|
|
499
524
|
this.attributes = attributes;
|
500
525
|
this.indexId = indexId;
|
501
526
|
this.xpaths = xpaths;
|
527
|
+
this.isVisible = isVisible;
|
502
528
|
}
|
503
529
|
};
|
504
530
|
|
@@ -521,14 +547,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
521
547
|
})
|
522
548
|
]);
|
523
549
|
const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
|
524
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
550
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
525
551
|
return new WebElementInfo({
|
526
552
|
rect,
|
527
553
|
locator,
|
528
554
|
id,
|
529
555
|
content,
|
530
556
|
attributes,
|
531
|
-
indexId
|
557
|
+
indexId,
|
558
|
+
isVisible
|
532
559
|
});
|
533
560
|
});
|
534
561
|
(0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
|
@@ -558,7 +585,7 @@ function printReportMsg(filepath) {
|
|
558
585
|
(0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
|
559
586
|
}
|
560
587
|
function replaceIllegalPathCharsAndSpace(str) {
|
561
|
-
return str.replace(/[
|
588
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
562
589
|
}
|
563
590
|
function matchElementFromPlan(planLocateParam, tree) {
|
564
591
|
if (!planLocateParam) {
|
@@ -587,6 +614,94 @@ var replanningCountLimit = 10;
|
|
587
614
|
var isAndroidPage = (page) => {
|
588
615
|
return page.pageType === "android";
|
589
616
|
};
|
617
|
+
var WorkflowMemory = class {
|
618
|
+
constructor(config) {
|
619
|
+
this.workflows = /* @__PURE__ */ new Map();
|
620
|
+
this.config = config;
|
621
|
+
}
|
622
|
+
/**
|
623
|
+
* İş akışı hafızasını getirir
|
624
|
+
*/
|
625
|
+
getWorkflowMemory(workflowId = "default") {
|
626
|
+
const workflow = this.workflows.get(workflowId);
|
627
|
+
return workflow?.memory || [];
|
628
|
+
}
|
629
|
+
/**
|
630
|
+
* İş akışı verilerini getirir
|
631
|
+
*/
|
632
|
+
getWorkflowData(workflowId = "default") {
|
633
|
+
return this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
634
|
+
}
|
635
|
+
/**
|
636
|
+
* İş akışı hafızasını kaydeder
|
637
|
+
*/
|
638
|
+
saveWorkflowMemory(memory, workflowId = "default") {
|
639
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
640
|
+
workflow.memory = [...memory];
|
641
|
+
workflow.metadata.totalSteps = workflow.steps.length;
|
642
|
+
workflow.metadata.completedSteps = workflow.steps.filter((s) => s.status === "completed").length;
|
643
|
+
workflow.metadata.failedSteps = workflow.steps.filter((s) => s.status === "failed").length;
|
644
|
+
this.workflows.set(workflowId, workflow);
|
645
|
+
this.enforceRetentionPolicy();
|
646
|
+
}
|
647
|
+
/**
|
648
|
+
* İş akışı bağlamını günceller
|
649
|
+
*/
|
650
|
+
updateWorkflowContext(context, workflowId = "default") {
|
651
|
+
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
652
|
+
workflow.context = { ...workflow.context, ...context };
|
653
|
+
if (context.currentStep) {
|
654
|
+
const existingStep = workflow.steps.find((s) => s.stepName === context.currentStep);
|
655
|
+
if (!existingStep) {
|
656
|
+
workflow.steps.push({
|
657
|
+
stepId: `step_${workflow.steps.length + 1}`,
|
658
|
+
stepName: context.currentStep,
|
659
|
+
timestamp: context.timestamp,
|
660
|
+
status: "running",
|
661
|
+
memoryItems: []
|
662
|
+
});
|
663
|
+
}
|
664
|
+
}
|
665
|
+
this.workflows.set(workflowId, workflow);
|
666
|
+
}
|
667
|
+
/**
|
668
|
+
* İş akışını temizler
|
669
|
+
*/
|
670
|
+
clearWorkflow(workflowId = "default") {
|
671
|
+
this.workflows.delete(workflowId);
|
672
|
+
}
|
673
|
+
/**
|
674
|
+
* Tüm iş akışlarını temizler
|
675
|
+
*/
|
676
|
+
clearAll() {
|
677
|
+
this.workflows.clear();
|
678
|
+
}
|
679
|
+
createEmptyWorkflowData(workflowId) {
|
680
|
+
return {
|
681
|
+
workflowId,
|
682
|
+
steps: [],
|
683
|
+
memory: [],
|
684
|
+
context: {
|
685
|
+
pageInfo: { url: "", title: "" },
|
686
|
+
timestamp: Date.now()
|
687
|
+
},
|
688
|
+
metadata: {
|
689
|
+
totalSteps: 0,
|
690
|
+
completedSteps: 0,
|
691
|
+
failedSteps: 0,
|
692
|
+
startTime: Date.now()
|
693
|
+
}
|
694
|
+
};
|
695
|
+
}
|
696
|
+
enforceRetentionPolicy() {
|
697
|
+
const maxWorkflows = 10;
|
698
|
+
if (this.workflows.size > maxWorkflows) {
|
699
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime));
|
700
|
+
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
701
|
+
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
702
|
+
}
|
703
|
+
}
|
704
|
+
};
|
590
705
|
var PageTaskExecutor = class {
|
591
706
|
constructor(page, insight, opts) {
|
592
707
|
this.conversationHistory = [];
|
@@ -594,6 +709,25 @@ var PageTaskExecutor = class {
|
|
594
709
|
this.insight = insight;
|
595
710
|
this.taskCache = opts.taskCache;
|
596
711
|
this.onTaskStartCallback = opts?.onTaskStart;
|
712
|
+
this.memoryConfig = {
|
713
|
+
maxItems: 100,
|
714
|
+
maxAge: 2 * 60 * 60 * 1e3,
|
715
|
+
// 2 saat
|
716
|
+
enablePersistence: true,
|
717
|
+
enableAnalytics: true,
|
718
|
+
filterStrategy: "hybrid",
|
719
|
+
...opts?.memoryConfig
|
720
|
+
};
|
721
|
+
this.sessionContext = {
|
722
|
+
sessionId: opts?.sessionId || this.generateSessionId(),
|
723
|
+
workflowId: opts?.workflowId,
|
724
|
+
startTime: Date.now(),
|
725
|
+
pageInfo: {
|
726
|
+
url: "",
|
727
|
+
title: ""
|
728
|
+
}
|
729
|
+
};
|
730
|
+
this.workflowMemory = new WorkflowMemory(this.memoryConfig);
|
597
731
|
}
|
598
732
|
async recordScreenshot(timing) {
|
599
733
|
const base64 = await this.page.screenshotBase64();
|
@@ -806,8 +940,11 @@ var PageTaskExecutor = class {
|
|
806
940
|
insightDump = dump;
|
807
941
|
};
|
808
942
|
this.insight.onceDumpUpdatedFn = dumpCollector;
|
943
|
+
const memoryContext = this.getMemoryAsContext();
|
809
944
|
const assertion = await this.insight.assert(
|
810
|
-
assertPlan.param.assertion
|
945
|
+
assertPlan.param.assertion,
|
946
|
+
memoryContext
|
947
|
+
// Hafıza bağlamını geç
|
811
948
|
);
|
812
949
|
if (!assertion.pass) {
|
813
950
|
if (plan2.type === "Assert") {
|
@@ -844,10 +981,10 @@ var PageTaskExecutor = class {
|
|
844
981
|
if (!taskParam || !taskParam.value) {
|
845
982
|
return;
|
846
983
|
}
|
847
|
-
await this.page.keyboard.type(taskParam.value);
|
848
|
-
} else {
|
849
|
-
await this.page.keyboard.type(taskParam.value);
|
850
984
|
}
|
985
|
+
await this.page.keyboard.type(taskParam.value, {
|
986
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
987
|
+
});
|
851
988
|
}
|
852
989
|
};
|
853
990
|
tasks.push(taskActionInput);
|
@@ -876,6 +1013,22 @@ var PageTaskExecutor = class {
|
|
876
1013
|
}
|
877
1014
|
};
|
878
1015
|
tasks.push(taskActionTap);
|
1016
|
+
} else if (plan2.type === "RightClick") {
|
1017
|
+
const taskActionRightClick = {
|
1018
|
+
type: "Action",
|
1019
|
+
subType: "RightClick",
|
1020
|
+
thought: plan2.thought,
|
1021
|
+
locate: plan2.locate,
|
1022
|
+
executor: async (param, { element }) => {
|
1023
|
+
(0, import_utils6.assert)(element, "Element not found, cannot right click");
|
1024
|
+
await this.page.mouse.click(
|
1025
|
+
element.center[0],
|
1026
|
+
element.center[1],
|
1027
|
+
{ button: "right" }
|
1028
|
+
);
|
1029
|
+
}
|
1030
|
+
};
|
1031
|
+
tasks.push(taskActionRightClick);
|
879
1032
|
} else if (plan2.type === "Drag") {
|
880
1033
|
const taskActionDrag = {
|
881
1034
|
type: "Action",
|
@@ -1277,25 +1430,146 @@ var PageTaskExecutor = class {
|
|
1277
1430
|
};
|
1278
1431
|
return task;
|
1279
1432
|
}
|
1433
|
+
/**
|
1434
|
+
* Persistent executor'ı getirir veya oluşturur
|
1435
|
+
*/
|
1436
|
+
getPersistentExecutor() {
|
1437
|
+
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1438
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(this.sessionContext.workflowId);
|
1439
|
+
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1440
|
+
onTaskStart: this.onTaskStartCallback,
|
1441
|
+
initialMemory: previousMemory
|
1442
|
+
});
|
1443
|
+
}
|
1444
|
+
return this.persistentExecutor;
|
1445
|
+
}
|
1446
|
+
/**
|
1447
|
+
* Sayfa bağlamını günceller
|
1448
|
+
*/
|
1449
|
+
async updatePageContext() {
|
1450
|
+
try {
|
1451
|
+
if (this.page.url) {
|
1452
|
+
this.sessionContext.pageInfo.url = await this.page.url();
|
1453
|
+
}
|
1454
|
+
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1455
|
+
this.sessionContext.pageInfo.title = await this.page.title();
|
1456
|
+
}
|
1457
|
+
} catch (e) {
|
1458
|
+
}
|
1459
|
+
}
|
1460
|
+
/**
|
1461
|
+
* Hafızayı temizler
|
1462
|
+
*/
|
1463
|
+
clearMemory() {
|
1464
|
+
if (this.persistentExecutor) {
|
1465
|
+
this.persistentExecutor.clearMemory();
|
1466
|
+
}
|
1467
|
+
this.workflowMemory.clearWorkflow(this.sessionContext.workflowId || "default");
|
1468
|
+
}
|
1469
|
+
/**
|
1470
|
+
* Mevcut hafızayı döndürür
|
1471
|
+
*/
|
1472
|
+
getMemory() {
|
1473
|
+
return this.persistentExecutor?.getMemory() || [];
|
1474
|
+
}
|
1475
|
+
/**
|
1476
|
+
* İş akışı hafızasını döndürür
|
1477
|
+
*/
|
1478
|
+
getWorkflowMemory() {
|
1479
|
+
return this.workflowMemory.getWorkflowData(this.sessionContext.workflowId || "default");
|
1480
|
+
}
|
1481
|
+
/**
|
1482
|
+
* Hafıza istatistiklerini döndürür
|
1483
|
+
*/
|
1484
|
+
getMemoryStats() {
|
1485
|
+
return this.persistentExecutor?.getMemoryStats() || {
|
1486
|
+
totalItems: 0,
|
1487
|
+
analytics: { totalTasks: 0, memoryHits: 0, memoryMisses: 0, averageMemorySize: 0, memoryEffectiveness: 0 },
|
1488
|
+
config: this.memoryConfig
|
1489
|
+
};
|
1490
|
+
}
|
1491
|
+
/**
|
1492
|
+
* Hafıza konfigürasyonunu günceller
|
1493
|
+
*/
|
1494
|
+
updateMemoryConfig(config) {
|
1495
|
+
this.memoryConfig = { ...this.memoryConfig, ...config };
|
1496
|
+
if (this.persistentExecutor) {
|
1497
|
+
this.persistentExecutor.updateMemoryConfig(this.memoryConfig);
|
1498
|
+
}
|
1499
|
+
}
|
1500
|
+
/**
|
1501
|
+
* Oturum ID'sini oluşturur
|
1502
|
+
*/
|
1503
|
+
generateSessionId() {
|
1504
|
+
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
|
1505
|
+
}
|
1506
|
+
/**
|
1507
|
+
* Hafızayı bağlam olarak formatlar
|
1508
|
+
*/
|
1509
|
+
getMemoryAsContext() {
|
1510
|
+
const memory = this.getMemory();
|
1511
|
+
if (memory.length === 0) {
|
1512
|
+
return "";
|
1513
|
+
}
|
1514
|
+
const recentMemory = memory.slice(-5);
|
1515
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
1516
|
+
}
|
1280
1517
|
async runPlans(title, plans, opts) {
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
1518
|
+
await this.updatePageContext();
|
1519
|
+
const useMemory = opts?.useMemory !== false;
|
1520
|
+
let taskExecutor;
|
1521
|
+
if (useMemory) {
|
1522
|
+
taskExecutor = this.getPersistentExecutor();
|
1523
|
+
this.workflowMemory.updateWorkflowContext({
|
1524
|
+
currentStep: title,
|
1525
|
+
pageInfo: this.sessionContext.pageInfo,
|
1526
|
+
timestamp: Date.now()
|
1527
|
+
}, this.sessionContext.workflowId || "default");
|
1528
|
+
} else {
|
1529
|
+
taskExecutor = new import_misoai_core.Executor(title, {
|
1530
|
+
onTaskStart: this.onTaskStartCallback
|
1531
|
+
});
|
1532
|
+
}
|
1284
1533
|
const { tasks } = await this.convertPlanToExecutable(plans, opts);
|
1534
|
+
tasks.forEach((task) => {
|
1535
|
+
task.context = {
|
1536
|
+
...task.context,
|
1537
|
+
...this.sessionContext.pageInfo,
|
1538
|
+
workflowId: this.sessionContext.workflowId,
|
1539
|
+
sessionId: this.sessionContext.sessionId
|
1540
|
+
};
|
1541
|
+
});
|
1285
1542
|
await taskExecutor.append(tasks);
|
1286
1543
|
const result = await taskExecutor.flush();
|
1544
|
+
if (useMemory) {
|
1545
|
+
this.workflowMemory.saveWorkflowMemory(
|
1546
|
+
taskExecutor.getMemory(),
|
1547
|
+
this.sessionContext.workflowId || "default"
|
1548
|
+
);
|
1549
|
+
}
|
1287
1550
|
return {
|
1288
1551
|
output: result,
|
1289
1552
|
executor: taskExecutor
|
1290
1553
|
};
|
1291
1554
|
}
|
1292
1555
|
async action(userPrompt, actionContext, opts) {
|
1293
|
-
const
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1556
|
+
const useMemory = true;
|
1557
|
+
let taskExecutor;
|
1558
|
+
if (useMemory) {
|
1559
|
+
taskExecutor = this.getPersistentExecutor();
|
1560
|
+
} else {
|
1561
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
|
1562
|
+
onTaskStart: this.onTaskStartCallback
|
1563
|
+
});
|
1564
|
+
}
|
1565
|
+
const memoryContext = this.getMemoryAsContext();
|
1566
|
+
const initialLog = memoryContext ? memoryContext : void 0;
|
1567
|
+
let planningTask = this.planningTaskFromPrompt(userPrompt, initialLog, actionContext);
|
1297
1568
|
let replanCount = 0;
|
1298
1569
|
const logList = [];
|
1570
|
+
if (memoryContext) {
|
1571
|
+
logList.push(memoryContext);
|
1572
|
+
}
|
1299
1573
|
const yamlFlow = [];
|
1300
1574
|
while (planningTask) {
|
1301
1575
|
if (replanCount > replanningCountLimit) {
|
@@ -1404,16 +1678,22 @@ var PageTaskExecutor = class {
|
|
1404
1678
|
executor: taskExecutor
|
1405
1679
|
};
|
1406
1680
|
}
|
1407
|
-
async createTypeQueryTask(type, demand) {
|
1408
|
-
const
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1416
|
-
|
1681
|
+
async createTypeQueryTask(type, demand, opt) {
|
1682
|
+
const useMemory = true;
|
1683
|
+
let taskExecutor;
|
1684
|
+
if (useMemory) {
|
1685
|
+
taskExecutor = this.getPersistentExecutor();
|
1686
|
+
} else {
|
1687
|
+
taskExecutor = new import_misoai_core.Executor(
|
1688
|
+
taskTitleStr(
|
1689
|
+
type,
|
1690
|
+
typeof demand === "string" ? demand : JSON.stringify(demand)
|
1691
|
+
),
|
1692
|
+
{
|
1693
|
+
onTaskStart: this.onTaskStartCallback
|
1694
|
+
}
|
1695
|
+
);
|
1696
|
+
}
|
1417
1697
|
const queryTask = {
|
1418
1698
|
type: "Insight",
|
1419
1699
|
subType: type,
|
@@ -1435,7 +1715,13 @@ var PageTaskExecutor = class {
|
|
1435
1715
|
result: `${type}, ${demand}`
|
1436
1716
|
};
|
1437
1717
|
}
|
1438
|
-
const
|
1718
|
+
const memoryContext = this.getMemoryAsContext();
|
1719
|
+
const { data, usage } = await this.insight.extract(
|
1720
|
+
demandInput,
|
1721
|
+
opt,
|
1722
|
+
memoryContext
|
1723
|
+
// Hafıza bağlamını geç
|
1724
|
+
);
|
1439
1725
|
let outputResult = data;
|
1440
1726
|
if (ifTypeRestricted) {
|
1441
1727
|
(0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
|
@@ -1455,23 +1741,29 @@ var PageTaskExecutor = class {
|
|
1455
1741
|
executor: taskExecutor
|
1456
1742
|
};
|
1457
1743
|
}
|
1458
|
-
async query(demand) {
|
1459
|
-
return this.createTypeQueryTask("Query", demand);
|
1744
|
+
async query(demand, opt) {
|
1745
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1460
1746
|
}
|
1461
|
-
async boolean(prompt) {
|
1462
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1747
|
+
async boolean(prompt, opt) {
|
1748
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1463
1749
|
}
|
1464
|
-
async number(prompt) {
|
1465
|
-
return this.createTypeQueryTask("Number", prompt);
|
1750
|
+
async number(prompt, opt) {
|
1751
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1466
1752
|
}
|
1467
|
-
async string(prompt) {
|
1468
|
-
return this.createTypeQueryTask("String", prompt);
|
1753
|
+
async string(prompt, opt) {
|
1754
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1469
1755
|
}
|
1470
|
-
async assert(assertion) {
|
1756
|
+
async assert(assertion, memoryContext) {
|
1471
1757
|
const description = `assert: ${assertion}`;
|
1472
|
-
const
|
1473
|
-
|
1474
|
-
|
1758
|
+
const useMemory = true;
|
1759
|
+
let taskExecutor;
|
1760
|
+
if (useMemory) {
|
1761
|
+
taskExecutor = this.getPersistentExecutor();
|
1762
|
+
} else {
|
1763
|
+
taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
|
1764
|
+
onTaskStart: this.onTaskStartCallback
|
1765
|
+
});
|
1766
|
+
}
|
1475
1767
|
const assertionPlan = {
|
1476
1768
|
type: "Assert",
|
1477
1769
|
param: {
|
@@ -1601,7 +1893,7 @@ function buildPlans(type, locateParam, param) {
|
|
1601
1893
|
param: locateParam,
|
1602
1894
|
thought: ""
|
1603
1895
|
} : null;
|
1604
|
-
if (type === "Tap" || type === "Hover") {
|
1896
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1605
1897
|
(0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
|
1606
1898
|
(0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
|
1607
1899
|
const tapPlan = {
|
@@ -1681,7 +1973,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1681
1973
|
var import_semver = __toESM(require("semver"));
|
1682
1974
|
|
1683
1975
|
// package.json
|
1684
|
-
var version = "1.5.
|
1976
|
+
var version = "1.5.7";
|
1685
1977
|
|
1686
1978
|
// src/common/task-cache.ts
|
1687
1979
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1812,8 +2104,14 @@ cache file: ${cacheFile}`
|
|
1812
2104
|
return;
|
1813
2105
|
}
|
1814
2106
|
try {
|
2107
|
+
const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
|
2108
|
+
if (!(0, import_node_fs2.existsSync)(dir)) {
|
2109
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
2110
|
+
debug3("created cache directory: %s", dir);
|
2111
|
+
}
|
1815
2112
|
const yamlData = import_js_yaml3.default.dump(this.cache);
|
1816
2113
|
(0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
|
2114
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1817
2115
|
} catch (err) {
|
1818
2116
|
debug3(
|
1819
2117
|
"write cache to file failed, path: %s, error: %s",
|
@@ -2071,6 +2369,23 @@ var PageAgent = class {
|
|
2071
2369
|
metadata
|
2072
2370
|
};
|
2073
2371
|
}
|
2372
|
+
async aiRightClick(locatePrompt, opt) {
|
2373
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2374
|
+
locatePrompt,
|
2375
|
+
opt
|
2376
|
+
);
|
2377
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2378
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2379
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2380
|
+
plans,
|
2381
|
+
{ cacheable: opt?.cacheable }
|
2382
|
+
);
|
2383
|
+
const metadata = this.afterTaskRunning(executor);
|
2384
|
+
return {
|
2385
|
+
result: output,
|
2386
|
+
metadata
|
2387
|
+
};
|
2388
|
+
}
|
2074
2389
|
async aiInput(value, locatePrompt, opt) {
|
2075
2390
|
(0, import_utils12.assert)(
|
2076
2391
|
typeof value === "string",
|
@@ -2145,7 +2460,13 @@ var PageAgent = class {
|
|
2145
2460
|
metadata: metadata2
|
2146
2461
|
};
|
2147
2462
|
}
|
2148
|
-
const
|
2463
|
+
const memoryContext = this.getMemoryAsContext();
|
2464
|
+
const enhancedActionContext = this.opts.aiActionContext ? `${this.opts.aiActionContext}
|
2465
|
+
|
2466
|
+
Previous workflow steps:
|
2467
|
+
${memoryContext}` : memoryContext ? `Previous workflow steps:
|
2468
|
+
${memoryContext}` : void 0;
|
2469
|
+
const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, enhancedActionContext, {
|
2149
2470
|
cacheable
|
2150
2471
|
}));
|
2151
2472
|
if (this.taskCache && output?.yamlFlow && cacheable !== false) {
|
@@ -2291,8 +2612,9 @@ var PageAgent = class {
|
|
2291
2612
|
} catch (e) {
|
2292
2613
|
}
|
2293
2614
|
}
|
2615
|
+
const memoryContext = this.getMemoryAsContext();
|
2294
2616
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2295
|
-
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2617
|
+
const { output, executor } = await this.taskExecutor.assert(assertionWithContext, memoryContext);
|
2296
2618
|
const metadata = this.afterTaskRunning(executor, true);
|
2297
2619
|
if (output && opt?.keepRawResponse) {
|
2298
2620
|
return {
|
@@ -2502,9 +2824,72 @@ ${errors}`);
|
|
2502
2824
|
}
|
2503
2825
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2504
2826
|
}
|
2827
|
+
async logScreenshot(title, options) {
|
2828
|
+
const screenshotTitle = title || "untitled";
|
2829
|
+
const content = options?.content || "";
|
2830
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2831
|
+
if (screenshot) {
|
2832
|
+
const executionDump = {
|
2833
|
+
name: screenshotTitle,
|
2834
|
+
description: content,
|
2835
|
+
tasks: [{
|
2836
|
+
type: "Screenshot",
|
2837
|
+
subType: "log",
|
2838
|
+
status: "finished",
|
2839
|
+
executor: null,
|
2840
|
+
param: {
|
2841
|
+
title: screenshotTitle,
|
2842
|
+
content
|
2843
|
+
},
|
2844
|
+
output: {
|
2845
|
+
screenshot
|
2846
|
+
},
|
2847
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2848
|
+
timing: {
|
2849
|
+
start: Date.now(),
|
2850
|
+
end: Date.now(),
|
2851
|
+
cost: 0
|
2852
|
+
}
|
2853
|
+
}],
|
2854
|
+
sdkVersion: "1.0.0",
|
2855
|
+
logTime: Date.now(),
|
2856
|
+
model_name: "screenshot"
|
2857
|
+
};
|
2858
|
+
this.appendExecutionDump(executionDump);
|
2859
|
+
}
|
2860
|
+
}
|
2505
2861
|
async destroy() {
|
2506
2862
|
await this.page.destroy();
|
2507
2863
|
}
|
2864
|
+
/**
|
2865
|
+
* Hafızayı bağlam olarak formatlar
|
2866
|
+
*/
|
2867
|
+
getMemoryAsContext() {
|
2868
|
+
const memory = this.taskExecutor.getMemory();
|
2869
|
+
if (memory.length === 0) {
|
2870
|
+
return "";
|
2871
|
+
}
|
2872
|
+
const recentMemory = memory.slice(-5);
|
2873
|
+
return recentMemory.map((item) => `- ${item.summary}`).join("\n");
|
2874
|
+
}
|
2875
|
+
/**
|
2876
|
+
* Mevcut hafızayı döndürür
|
2877
|
+
*/
|
2878
|
+
getMemory() {
|
2879
|
+
return this.taskExecutor.getMemory();
|
2880
|
+
}
|
2881
|
+
/**
|
2882
|
+
* Hafıza istatistiklerini döndürür
|
2883
|
+
*/
|
2884
|
+
getMemoryStats() {
|
2885
|
+
return this.taskExecutor.getMemoryStats();
|
2886
|
+
}
|
2887
|
+
/**
|
2888
|
+
* Hafızayı temizler
|
2889
|
+
*/
|
2890
|
+
clearMemory() {
|
2891
|
+
this.taskExecutor.clearMemory();
|
2892
|
+
}
|
2508
2893
|
};
|
2509
2894
|
|
2510
2895
|
// src/bridge-mode/agent-cli-side.ts
|
@@ -2617,7 +3002,7 @@ var BridgeServer = class {
|
|
2617
3002
|
this.socket = socket;
|
2618
3003
|
const clientVersion = socket.handshake.query.version;
|
2619
3004
|
(0, import_utils16.logMsg)(
|
2620
|
-
`Bridge connected, cli-side version v${"1.5.
|
3005
|
+
`Bridge connected, cli-side version v${"1.5.7"}, browser-side version v${clientVersion}`
|
2621
3006
|
);
|
2622
3007
|
socket.on("bridge-call-response" /* CallResponse */, (params) => {
|
2623
3008
|
const id = params.id;
|
@@ -2648,7 +3033,7 @@ var BridgeServer = class {
|
|
2648
3033
|
setTimeout(() => {
|
2649
3034
|
this.onConnect?.();
|
2650
3035
|
const payload = {
|
2651
|
-
version: "1.5.
|
3036
|
+
version: "1.5.7"
|
2652
3037
|
};
|
2653
3038
|
socket.emit("bridge-connected" /* Connected */, payload);
|
2654
3039
|
Promise.resolve().then(() => {
|