misoai-web 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +108 -50
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +110 -52
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +109 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +425 -68
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +111 -53
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js +4 -4
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +108 -50
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +425 -68
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +424 -67
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +424 -67
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/utils.js +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +108 -50
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +110 -52
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +109 -51
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +425 -68
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +111 -53
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js +4 -4
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +108 -50
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +425 -68
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +424 -67
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +424 -67
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +1 -1
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-9b472ffb.d.ts → browser-f205f69d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +2 -2
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-ed0ecb44.d.ts → page-c5452809.d.ts} +45 -0
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +2 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +2 -2
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +18 -54
package/dist/lib/playwright.js
CHANGED
@@ -345,8 +345,8 @@ var ScriptPlayer = class {
|
|
345
345
|
var import_js_yaml = __toESM(require("js-yaml"));
|
346
346
|
|
347
347
|
// src/yaml/utils.ts
|
348
|
-
var import_utils2 = require("misoai-shared/utils");
|
349
348
|
var import_js_yaml2 = __toESM(require("js-yaml"));
|
349
|
+
var import_utils2 = require("misoai-shared/utils");
|
350
350
|
function interpolateEnvVars(content) {
|
351
351
|
return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
|
352
352
|
const value = process.env[envVar.trim()];
|
@@ -492,13 +492,13 @@ function paramStr(task) {
|
|
492
492
|
}
|
493
493
|
|
494
494
|
// src/common/utils.ts
|
495
|
+
var import_dayjs = __toESM(require("dayjs"));
|
495
496
|
var import_ai_model = require("misoai-core/ai-model");
|
496
497
|
var import_utils3 = require("misoai-core/utils");
|
497
498
|
var import_env = require("misoai-shared/env");
|
498
499
|
var import_extractor = require("misoai-shared/extractor");
|
499
500
|
var import_img = require("misoai-shared/img");
|
500
501
|
var import_utils4 = require("misoai-shared/utils");
|
501
|
-
var import_dayjs = __toESM(require("dayjs"));
|
502
502
|
|
503
503
|
// src/web-element.ts
|
504
504
|
var WebElementInfo = class {
|
@@ -667,8 +667,12 @@ var WorkflowMemory = class {
|
|
667
667
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
668
668
|
workflow.memory = [...memory];
|
669
669
|
workflow.metadata.totalSteps = workflow.steps.length;
|
670
|
-
workflow.metadata.completedSteps = workflow.steps.filter(
|
671
|
-
|
670
|
+
workflow.metadata.completedSteps = workflow.steps.filter(
|
671
|
+
(s) => s.status === "completed"
|
672
|
+
).length;
|
673
|
+
workflow.metadata.failedSteps = workflow.steps.filter(
|
674
|
+
(s) => s.status === "failed"
|
675
|
+
).length;
|
672
676
|
this.workflows.set(workflowId, workflow);
|
673
677
|
this.enforceRetentionPolicy();
|
674
678
|
}
|
@@ -679,7 +683,9 @@ var WorkflowMemory = class {
|
|
679
683
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
680
684
|
workflow.context = { ...workflow.context, ...context };
|
681
685
|
if (context.currentStep) {
|
682
|
-
const existingStep = workflow.steps.find(
|
686
|
+
const existingStep = workflow.steps.find(
|
687
|
+
(s) => s.stepName === context.currentStep
|
688
|
+
);
|
683
689
|
if (!existingStep) {
|
684
690
|
workflow.steps.push({
|
685
691
|
stepId: `step_${workflow.steps.length + 1}`,
|
@@ -724,7 +730,9 @@ var WorkflowMemory = class {
|
|
724
730
|
enforceRetentionPolicy() {
|
725
731
|
const maxWorkflows = 10;
|
726
732
|
if (this.workflows.size > maxWorkflows) {
|
727
|
-
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
733
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
734
|
+
([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
|
735
|
+
);
|
728
736
|
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
729
737
|
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
730
738
|
}
|
@@ -1291,6 +1299,7 @@ var PageTaskExecutor = class {
|
|
1291
1299
|
actions: [],
|
1292
1300
|
more_actions_needed_by_instruction: false,
|
1293
1301
|
log: "",
|
1302
|
+
summary: "Loaded YAML workflow configuration",
|
1294
1303
|
yamlString
|
1295
1304
|
},
|
1296
1305
|
cache: {
|
@@ -1392,6 +1401,7 @@ var PageTaskExecutor = class {
|
|
1392
1401
|
actions: finalActions,
|
1393
1402
|
more_actions_needed_by_instruction,
|
1394
1403
|
log: log2,
|
1404
|
+
summary: planResult.summary || "Generated action plan from user instruction",
|
1395
1405
|
yamlFlow: planResult.yamlFlow
|
1396
1406
|
},
|
1397
1407
|
cache: {
|
@@ -1447,6 +1457,7 @@ var PageTaskExecutor = class {
|
|
1447
1457
|
actionType: actions[0].type,
|
1448
1458
|
more_actions_needed_by_instruction: true,
|
1449
1459
|
log: "",
|
1460
|
+
summary: action_summary || "Generated VLM action plan",
|
1450
1461
|
yamlFlow: planResult.yamlFlow
|
1451
1462
|
},
|
1452
1463
|
cache: {
|
@@ -1463,7 +1474,9 @@ var PageTaskExecutor = class {
|
|
1463
1474
|
*/
|
1464
1475
|
getPersistentExecutor() {
|
1465
1476
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1466
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1477
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1478
|
+
this.sessionContext.workflowId
|
1479
|
+
);
|
1467
1480
|
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1468
1481
|
onTaskStart: this.onTaskStartCallback,
|
1469
1482
|
initialMemory: previousMemory
|
@@ -1492,7 +1505,9 @@ var PageTaskExecutor = class {
|
|
1492
1505
|
if (this.persistentExecutor) {
|
1493
1506
|
this.persistentExecutor.clearMemory();
|
1494
1507
|
}
|
1495
|
-
this.workflowMemory.clearWorkflow(
|
1508
|
+
this.workflowMemory.clearWorkflow(
|
1509
|
+
this.sessionContext.workflowId || "default"
|
1510
|
+
);
|
1496
1511
|
}
|
1497
1512
|
/**
|
1498
1513
|
* Mevcut hafızayı döndürür
|
@@ -1504,7 +1519,9 @@ var PageTaskExecutor = class {
|
|
1504
1519
|
* İş akışı hafızasını döndürür
|
1505
1520
|
*/
|
1506
1521
|
getWorkflowMemory() {
|
1507
|
-
return this.workflowMemory.getWorkflowData(
|
1522
|
+
return this.workflowMemory.getWorkflowData(
|
1523
|
+
this.sessionContext.workflowId || "default"
|
1524
|
+
);
|
1508
1525
|
}
|
1509
1526
|
/**
|
1510
1527
|
* Hafıza istatistiklerini döndürür
|
@@ -1512,7 +1529,13 @@ var PageTaskExecutor = class {
|
|
1512
1529
|
getMemoryStats() {
|
1513
1530
|
return this.persistentExecutor?.getMemoryStats() || {
|
1514
1531
|
totalItems: 0,
|
1515
|
-
analytics: {
|
1532
|
+
analytics: {
|
1533
|
+
totalTasks: 0,
|
1534
|
+
memoryHits: 0,
|
1535
|
+
memoryMisses: 0,
|
1536
|
+
averageMemorySize: 0,
|
1537
|
+
memoryEffectiveness: 0
|
1538
|
+
},
|
1516
1539
|
config: this.memoryConfig
|
1517
1540
|
};
|
1518
1541
|
}
|
@@ -1548,11 +1571,14 @@ var PageTaskExecutor = class {
|
|
1548
1571
|
let taskExecutor;
|
1549
1572
|
if (useMemory) {
|
1550
1573
|
taskExecutor = this.getPersistentExecutor();
|
1551
|
-
this.workflowMemory.updateWorkflowContext(
|
1552
|
-
|
1553
|
-
|
1554
|
-
|
1555
|
-
|
1574
|
+
this.workflowMemory.updateWorkflowContext(
|
1575
|
+
{
|
1576
|
+
currentStep: title,
|
1577
|
+
pageInfo: this.sessionContext.pageInfo,
|
1578
|
+
timestamp: Date.now()
|
1579
|
+
},
|
1580
|
+
this.sessionContext.workflowId || "default"
|
1581
|
+
);
|
1556
1582
|
} else {
|
1557
1583
|
taskExecutor = new import_misoai_core.Executor(title, {
|
1558
1584
|
onTaskStart: this.onTaskStartCallback
|
@@ -1918,14 +1944,19 @@ var PageTaskExecutor = class {
|
|
1918
1944
|
*/
|
1919
1945
|
addToMemory(memoryItem) {
|
1920
1946
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1921
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1947
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1948
|
+
this.sessionContext.workflowId
|
1949
|
+
);
|
1922
1950
|
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1923
1951
|
onTaskStart: this.onTaskStartCallback,
|
1924
1952
|
initialMemory: previousMemory
|
1925
1953
|
});
|
1926
1954
|
}
|
1927
1955
|
this.persistentExecutor.memoryStore?.add(memoryItem);
|
1928
|
-
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1956
|
+
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1957
|
+
"add",
|
1958
|
+
memoryItem
|
1959
|
+
);
|
1929
1960
|
}
|
1930
1961
|
};
|
1931
1962
|
|
@@ -2014,14 +2045,14 @@ function buildPlans(type, locateParam, param) {
|
|
2014
2045
|
var import_node_assert = __toESM(require("assert"));
|
2015
2046
|
var import_node_fs2 = require("fs");
|
2016
2047
|
var import_node_path2 = require("path");
|
2048
|
+
var import_js_yaml3 = __toESM(require("js-yaml"));
|
2017
2049
|
var import_common2 = require("misoai-shared/common");
|
2018
2050
|
var import_logger3 = require("misoai-shared/logger");
|
2019
2051
|
var import_utils9 = require("misoai-shared/utils");
|
2020
|
-
var import_js_yaml3 = __toESM(require("js-yaml"));
|
2021
2052
|
var import_semver = __toESM(require("semver"));
|
2022
2053
|
|
2023
2054
|
// package.json
|
2024
|
-
var version = "1.6.
|
2055
|
+
var version = "1.6.2";
|
2025
2056
|
|
2026
2057
|
// src/common/task-cache.ts
|
2027
2058
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -2316,22 +2347,28 @@ var PageAgent = class {
|
|
2316
2347
|
const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
|
2317
2348
|
const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
|
2318
2349
|
const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
|
2319
|
-
const planningTasks = executor.tasks.filter(
|
2320
|
-
|
2350
|
+
const planningTasks = executor.tasks.filter(
|
2351
|
+
(task) => task.type === "Planning"
|
2352
|
+
);
|
2353
|
+
const insightTasks = executor.tasks.filter(
|
2354
|
+
(task) => task.type === "Insight"
|
2355
|
+
);
|
2321
2356
|
const actionTasks = executor.tasks.filter((task) => task.type === "Action");
|
2322
2357
|
const planning = planningTasks.length > 0 ? {
|
2323
2358
|
type: "Planning",
|
2324
|
-
description:
|
2359
|
+
description: "Planning for task execution",
|
2325
2360
|
steps: planningTasks.map((task) => task.thought || "Planning step")
|
2326
2361
|
} : void 0;
|
2327
2362
|
const insight = insightTasks.length > 0 ? {
|
2328
2363
|
type: "Insight",
|
2329
|
-
description:
|
2330
|
-
elements: insightTasks.map(
|
2364
|
+
description: "Insight for task execution",
|
2365
|
+
elements: insightTasks.map(
|
2366
|
+
(task) => task.thought || "Insight element"
|
2367
|
+
)
|
2331
2368
|
} : void 0;
|
2332
2369
|
const action = actionTasks.length > 0 ? {
|
2333
2370
|
type: "Action",
|
2334
|
-
description:
|
2371
|
+
description: "Action for task execution",
|
2335
2372
|
result: lastTask?.output
|
2336
2373
|
} : void 0;
|
2337
2374
|
const actionDetails = executor.tasks.map((task) => ({
|
@@ -2665,7 +2702,10 @@ ${memoryContext}` : void 0;
|
|
2665
2702
|
}
|
2666
2703
|
const memoryContext = this.getMemoryAsContext();
|
2667
2704
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2668
|
-
const { output, executor } = await this.taskExecutor.assert(
|
2705
|
+
const { output, executor } = await this.taskExecutor.assert(
|
2706
|
+
assertionWithContext,
|
2707
|
+
memoryContext
|
2708
|
+
);
|
2669
2709
|
const metadata = this.afterTaskRunning(executor, true);
|
2670
2710
|
if (output && opt?.keepRawResponse) {
|
2671
2711
|
return {
|
@@ -2704,7 +2744,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
|
|
2704
2744
|
Return only "complex" or "simple" based on your analysis.
|
2705
2745
|
`;
|
2706
2746
|
const complexityMsgs = [
|
2707
|
-
{
|
2747
|
+
{
|
2748
|
+
role: "system",
|
2749
|
+
content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
|
2750
|
+
},
|
2708
2751
|
{
|
2709
2752
|
role: "user",
|
2710
2753
|
content: [
|
@@ -2728,7 +2771,12 @@ Return only "complex" or "simple" based on your analysis.
|
|
2728
2771
|
);
|
2729
2772
|
const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
|
2730
2773
|
shouldUseDeepThink = responseText.includes("complex");
|
2731
|
-
debug4(
|
2774
|
+
debug4(
|
2775
|
+
"CAPTCHA complexity analysis:",
|
2776
|
+
responseText,
|
2777
|
+
"Using deep think:",
|
2778
|
+
shouldUseDeepThink
|
2779
|
+
);
|
2732
2780
|
} catch (error) {
|
2733
2781
|
debug4("Failed to analyze CAPTCHA complexity:", error);
|
2734
2782
|
}
|
@@ -2745,7 +2793,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2745
2793
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2746
2794
|
} else if (action.type === "input" && action.value) {
|
2747
2795
|
if (action.target) {
|
2748
|
-
await this.aiInput(action.value, action.target, {
|
2796
|
+
await this.aiInput(action.value, action.target, {
|
2797
|
+
deepThink: shouldUseDeepThink
|
2798
|
+
});
|
2749
2799
|
}
|
2750
2800
|
} else if (action.type === "verify" && action.target) {
|
2751
2801
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
@@ -2757,7 +2807,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2757
2807
|
if (action.coordinates) {
|
2758
2808
|
const x = action.coordinates[0];
|
2759
2809
|
const y = action.coordinates[1];
|
2760
|
-
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2810
|
+
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2811
|
+
deepThink: shouldUseDeepThink
|
2812
|
+
});
|
2761
2813
|
} else if (action.target) {
|
2762
2814
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2763
2815
|
}
|
@@ -2909,25 +2961,27 @@ ${errors}`);
|
|
2909
2961
|
const executionDump = {
|
2910
2962
|
name: screenshotTitle,
|
2911
2963
|
description: content,
|
2912
|
-
tasks: [
|
2913
|
-
|
2914
|
-
|
2915
|
-
|
2916
|
-
|
2917
|
-
|
2918
|
-
|
2919
|
-
|
2920
|
-
|
2921
|
-
|
2922
|
-
|
2923
|
-
|
2924
|
-
|
2925
|
-
|
2926
|
-
|
2927
|
-
|
2928
|
-
|
2964
|
+
tasks: [
|
2965
|
+
{
|
2966
|
+
type: "Screenshot",
|
2967
|
+
subType: "log",
|
2968
|
+
status: "finished",
|
2969
|
+
executor: null,
|
2970
|
+
param: {
|
2971
|
+
title: screenshotTitle,
|
2972
|
+
content
|
2973
|
+
},
|
2974
|
+
output: {
|
2975
|
+
screenshot
|
2976
|
+
},
|
2977
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2978
|
+
timing: {
|
2979
|
+
start: Date.now(),
|
2980
|
+
end: Date.now(),
|
2981
|
+
cost: 0
|
2982
|
+
}
|
2929
2983
|
}
|
2930
|
-
|
2984
|
+
],
|
2931
2985
|
sdkVersion: "1.0.0",
|
2932
2986
|
logTime: Date.now(),
|
2933
2987
|
model_name: "screenshot"
|
@@ -2979,7 +3033,9 @@ ${errors}`);
|
|
2979
3033
|
totalTasks: stats.analytics.totalTasks,
|
2980
3034
|
memoryHits: stats.analytics.memoryHits,
|
2981
3035
|
memoryMisses: stats.analytics.memoryMisses,
|
2982
|
-
memoryEffectiveness: Math.round(
|
3036
|
+
memoryEffectiveness: Math.round(
|
3037
|
+
stats.analytics.memoryEffectiveness * 100
|
3038
|
+
),
|
2983
3039
|
averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
|
2984
3040
|
},
|
2985
3041
|
config: stats.config,
|
@@ -3043,7 +3099,9 @@ ${errors}`);
|
|
3043
3099
|
calculateSuccessRate(memory) {
|
3044
3100
|
if (memory.length === 0)
|
3045
3101
|
return 0;
|
3046
|
-
const successCount = memory.filter(
|
3102
|
+
const successCount = memory.filter(
|
3103
|
+
(item) => item.metadata?.success !== false
|
3104
|
+
).length;
|
3047
3105
|
return Math.round(successCount / memory.length * 100);
|
3048
3106
|
}
|
3049
3107
|
calculateAverageExecutionTime(memory) {
|
@@ -3079,13 +3137,236 @@ var import_extractor2 = require("misoai-shared/extractor");
|
|
3079
3137
|
var import_fs = require("misoai-shared/fs");
|
3080
3138
|
var import_logger5 = require("misoai-shared/logger");
|
3081
3139
|
var import_utils16 = require("misoai-shared/utils");
|
3140
|
+
|
3141
|
+
// src/common/mouse-utils.ts
|
3142
|
+
function generateHumanMousePath(from, to, options = {}) {
|
3143
|
+
const {
|
3144
|
+
steps = 20,
|
3145
|
+
easing = "easeInOut"
|
3146
|
+
} = options;
|
3147
|
+
if (steps <= 1) {
|
3148
|
+
return [to];
|
3149
|
+
}
|
3150
|
+
const path = [];
|
3151
|
+
const controlPoint1 = {
|
3152
|
+
x: from.x + (to.x - from.x) * 0.25 + (Math.random() - 0.5) * 50,
|
3153
|
+
y: from.y + (to.y - from.y) * 0.25 + (Math.random() - 0.5) * 50
|
3154
|
+
};
|
3155
|
+
const controlPoint2 = {
|
3156
|
+
x: from.x + (to.x - from.x) * 0.75 + (Math.random() - 0.5) * 50,
|
3157
|
+
y: from.y + (to.y - from.y) * 0.75 + (Math.random() - 0.5) * 50
|
3158
|
+
};
|
3159
|
+
for (let i = 0; i <= steps; i++) {
|
3160
|
+
const t = i / steps;
|
3161
|
+
const easedT = applyEasing(t, easing);
|
3162
|
+
const point = calculateBezierPoint(from, controlPoint1, controlPoint2, to, easedT);
|
3163
|
+
path.push(point);
|
3164
|
+
}
|
3165
|
+
return path;
|
3166
|
+
}
|
3167
|
+
function calculateBezierPoint(p0, p1, p2, p3, t) {
|
3168
|
+
const oneMinusT = 1 - t;
|
3169
|
+
const oneMinusTSquared = oneMinusT * oneMinusT;
|
3170
|
+
const oneMinusTCubed = oneMinusTSquared * oneMinusT;
|
3171
|
+
const tSquared = t * t;
|
3172
|
+
const tCubed = tSquared * t;
|
3173
|
+
return {
|
3174
|
+
x: oneMinusTCubed * p0.x + 3 * oneMinusTSquared * t * p1.x + 3 * oneMinusT * tSquared * p2.x + tCubed * p3.x,
|
3175
|
+
y: oneMinusTCubed * p0.y + 3 * oneMinusTSquared * t * p1.y + 3 * oneMinusT * tSquared * p2.y + tCubed * p3.y
|
3176
|
+
};
|
3177
|
+
}
|
3178
|
+
function applyEasing(t, easing) {
|
3179
|
+
switch (easing) {
|
3180
|
+
case "linear":
|
3181
|
+
return t;
|
3182
|
+
case "easeIn":
|
3183
|
+
return t * t;
|
3184
|
+
case "easeOut":
|
3185
|
+
return 1 - (1 - t) * (1 - t);
|
3186
|
+
case "easeInOut":
|
3187
|
+
return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
|
3188
|
+
default:
|
3189
|
+
return t;
|
3190
|
+
}
|
3191
|
+
}
|
3192
|
+
function calculateStepDelay(totalDuration, currentStep, totalSteps) {
|
3193
|
+
const baseDelay = totalDuration / totalSteps;
|
3194
|
+
const variation = 0.3;
|
3195
|
+
const randomFactor = 1 + (Math.random() - 0.5) * variation;
|
3196
|
+
return Math.max(1, baseDelay * randomFactor);
|
3197
|
+
}
|
3198
|
+
|
3199
|
+
// src/common/visual-feedback.ts
|
3200
|
+
function getMousePointerScript(x, y, options = {}) {
|
3201
|
+
const {
|
3202
|
+
pointerSize = 20,
|
3203
|
+
pointerColor = "#ff4444",
|
3204
|
+
trailLength = 5,
|
3205
|
+
showTrail = true,
|
3206
|
+
animationDuration = 200
|
3207
|
+
} = options;
|
3208
|
+
return `
|
3209
|
+
(() => {
|
3210
|
+
// Create or update mouse pointer
|
3211
|
+
if (!window.misoaiMousePointer) {
|
3212
|
+
const pointer = document.createElement('div');
|
3213
|
+
pointer.id = 'misoai-mouse-pointer';
|
3214
|
+
pointer.style.cssText = \`
|
3215
|
+
position: fixed;
|
3216
|
+
width: ${pointerSize}px;
|
3217
|
+
height: ${pointerSize}px;
|
3218
|
+
background: ${pointerColor};
|
3219
|
+
border: 2px solid white;
|
3220
|
+
border-radius: 50%;
|
3221
|
+
pointer-events: none;
|
3222
|
+
z-index: 999999;
|
3223
|
+
box-shadow: 0 0 10px rgba(0,0,0,0.3);
|
3224
|
+
transition: all ${animationDuration}ms ease-out;
|
3225
|
+
transform: translate(-50%, -50%);
|
3226
|
+
\`;
|
3227
|
+
document.body.appendChild(pointer);
|
3228
|
+
window.misoaiMousePointer = pointer;
|
3229
|
+
|
3230
|
+
// Initialize trail array
|
3231
|
+
window.misoaiMouseTrail = [];
|
3232
|
+
}
|
3233
|
+
|
3234
|
+
const pointer = window.misoaiMousePointer;
|
3235
|
+
pointer.style.left = ${x}px;
|
3236
|
+
pointer.style.top = ${y}px;
|
3237
|
+
|
3238
|
+
${showTrail ? getTrailScript(x, y, trailLength, pointerColor) : ""}
|
3239
|
+
|
3240
|
+
// Auto-hide after 3 seconds of inactivity
|
3241
|
+
clearTimeout(window.misoaiMouseTimeout);
|
3242
|
+
pointer.style.opacity = '1';
|
3243
|
+
window.misoaiMouseTimeout = setTimeout(() => {
|
3244
|
+
if (pointer) pointer.style.opacity = '0.3';
|
3245
|
+
}, 3000);
|
3246
|
+
})();
|
3247
|
+
`;
|
3248
|
+
}
|
3249
|
+
function getTrailScript(x, y, trailLength, color) {
|
3250
|
+
return `
|
3251
|
+
// Add current position to trail
|
3252
|
+
window.misoaiMouseTrail.push({ x: ${x}, y: ${y}, timestamp: Date.now() });
|
3253
|
+
|
3254
|
+
// Keep only recent trail points
|
3255
|
+
if (window.misoaiMouseTrail.length > ${trailLength}) {
|
3256
|
+
window.misoaiMouseTrail.shift();
|
3257
|
+
}
|
3258
|
+
|
3259
|
+
// Remove old trail elements
|
3260
|
+
document.querySelectorAll('.misoai-mouse-trail').forEach(el => el.remove());
|
3261
|
+
|
3262
|
+
// Create new trail elements
|
3263
|
+
window.misoaiMouseTrail.forEach((point, index) => {
|
3264
|
+
if (index === window.misoaiMouseTrail.length - 1) return; // Skip current position
|
3265
|
+
|
3266
|
+
const trailElement = document.createElement('div');
|
3267
|
+
trailElement.className = 'misoai-mouse-trail';
|
3268
|
+
const opacity = (index + 1) / window.misoaiMouseTrail.length * 0.6;
|
3269
|
+
const size = 8 + (index + 1) / window.misoaiMouseTrail.length * 8;
|
3270
|
+
|
3271
|
+
trailElement.style.cssText = \`
|
3272
|
+
position: fixed;
|
3273
|
+
width: \${size}px;
|
3274
|
+
height: \${size}px;
|
3275
|
+
background: ${color};
|
3276
|
+
border-radius: 50%;
|
3277
|
+
pointer-events: none;
|
3278
|
+
z-index: 999998;
|
3279
|
+
opacity: \${opacity};
|
3280
|
+
transform: translate(-50%, -50%);
|
3281
|
+
left: \${point.x}px;
|
3282
|
+
top: \${point.y}px;
|
3283
|
+
\`;
|
3284
|
+
|
3285
|
+
document.body.appendChild(trailElement);
|
3286
|
+
|
3287
|
+
// Auto-remove trail element after animation
|
3288
|
+
setTimeout(() => {
|
3289
|
+
if (trailElement.parentNode) {
|
3290
|
+
trailElement.parentNode.removeChild(trailElement);
|
3291
|
+
}
|
3292
|
+
}, 1000);
|
3293
|
+
});
|
3294
|
+
`;
|
3295
|
+
}
|
3296
|
+
function getClickAnimationScript(x, y) {
|
3297
|
+
return `
|
3298
|
+
(() => {
|
3299
|
+
const clickRipple = document.createElement('div');
|
3300
|
+
clickRipple.style.cssText = \`
|
3301
|
+
position: fixed;
|
3302
|
+
left: ${x}px;
|
3303
|
+
top: ${y}px;
|
3304
|
+
width: 0;
|
3305
|
+
height: 0;
|
3306
|
+
border: 2px solid #ff4444;
|
3307
|
+
border-radius: 50%;
|
3308
|
+
pointer-events: none;
|
3309
|
+
z-index: 999999;
|
3310
|
+
transform: translate(-50%, -50%);
|
3311
|
+
animation: misoai-click-ripple 0.6s ease-out forwards;
|
3312
|
+
\`;
|
3313
|
+
|
3314
|
+
// Add CSS animation if not exists
|
3315
|
+
if (!document.getElementById('misoai-click-styles')) {
|
3316
|
+
const style = document.createElement('style');
|
3317
|
+
style.id = 'misoai-click-styles';
|
3318
|
+
style.textContent = \`
|
3319
|
+
@keyframes misoai-click-ripple {
|
3320
|
+
0% {
|
3321
|
+
width: 0;
|
3322
|
+
height: 0;
|
3323
|
+
opacity: 1;
|
3324
|
+
}
|
3325
|
+
100% {
|
3326
|
+
width: 40px;
|
3327
|
+
height: 40px;
|
3328
|
+
opacity: 0;
|
3329
|
+
}
|
3330
|
+
}
|
3331
|
+
\`;
|
3332
|
+
document.head.appendChild(style);
|
3333
|
+
}
|
3334
|
+
|
3335
|
+
document.body.appendChild(clickRipple);
|
3336
|
+
|
3337
|
+
// Remove after animation
|
3338
|
+
setTimeout(() => {
|
3339
|
+
if (clickRipple.parentNode) {
|
3340
|
+
clickRipple.parentNode.removeChild(clickRipple);
|
3341
|
+
}
|
3342
|
+
}, 600);
|
3343
|
+
})();
|
3344
|
+
`;
|
3345
|
+
}
|
3346
|
+
|
3347
|
+
// src/puppeteer/base-page.ts
|
3082
3348
|
var debugPage = (0, import_logger5.getDebug)("web:page");
|
3083
3349
|
var Page = class {
|
3084
3350
|
constructor(underlyingPage, pageType, opts) {
|
3351
|
+
this.currentMousePosition = { x: 0, y: 0 };
|
3352
|
+
this.mouseMovementOptions = {
|
3353
|
+
steps: 15,
|
3354
|
+
duration: 300,
|
3355
|
+
easing: "easeInOut",
|
3356
|
+
showVisualFeedback: true
|
3357
|
+
};
|
3358
|
+
this.visualFeedbackOptions = {
|
3359
|
+
pointerSize: 16,
|
3360
|
+
pointerColor: "#ff4444",
|
3361
|
+
trailLength: 4,
|
3362
|
+
showTrail: true,
|
3363
|
+
animationDuration: 150
|
3364
|
+
};
|
3085
3365
|
this.everMoved = false;
|
3086
3366
|
this.underlyingPage = underlyingPage;
|
3087
3367
|
this.pageType = pageType;
|
3088
3368
|
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
3369
|
+
this.initializeMousePosition();
|
3089
3370
|
}
|
3090
3371
|
async evaluate(pageFunction, arg) {
|
3091
3372
|
let result;
|
@@ -3104,9 +3385,76 @@ var Page = class {
|
|
3104
3385
|
debugPage("evaluate function end");
|
3105
3386
|
return result;
|
3106
3387
|
}
|
3388
|
+
/**
|
3389
|
+
* Initialize mouse position to center of viewport
|
3390
|
+
*/
|
3391
|
+
async initializeMousePosition() {
|
3392
|
+
try {
|
3393
|
+
const size = await this.size();
|
3394
|
+
this.currentMousePosition = {
|
3395
|
+
x: Math.floor(size.width / 2),
|
3396
|
+
y: Math.floor(size.height / 2)
|
3397
|
+
};
|
3398
|
+
} catch (error) {
|
3399
|
+
this.currentMousePosition = { x: 400, y: 300 };
|
3400
|
+
}
|
3401
|
+
}
|
3107
3402
|
async evaluateJavaScript(script) {
|
3108
3403
|
return this.evaluate(script);
|
3109
3404
|
}
|
3405
|
+
/**
|
3406
|
+
* Show visual feedback for mouse pointer
|
3407
|
+
*/
|
3408
|
+
async showMousePointer(x, y) {
|
3409
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3410
|
+
return;
|
3411
|
+
const script = getMousePointerScript(x, y, this.visualFeedbackOptions);
|
3412
|
+
try {
|
3413
|
+
await this.evaluate(script);
|
3414
|
+
} catch (error) {
|
3415
|
+
debugPage("Visual feedback error:", error);
|
3416
|
+
}
|
3417
|
+
}
|
3418
|
+
/**
|
3419
|
+
* Show click animation
|
3420
|
+
*/
|
3421
|
+
async showClickAnimation(x, y) {
|
3422
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3423
|
+
return;
|
3424
|
+
const script = getClickAnimationScript(x, y);
|
3425
|
+
try {
|
3426
|
+
await this.evaluate(script);
|
3427
|
+
} catch (error) {
|
3428
|
+
debugPage("Click animation error:", error);
|
3429
|
+
}
|
3430
|
+
}
|
3431
|
+
/**
|
3432
|
+
* Move mouse with human-like movement and visual feedback
|
3433
|
+
*/
|
3434
|
+
async moveMouseHumanLike(toX, toY, options) {
|
3435
|
+
const moveOptions = { ...this.mouseMovementOptions, ...options };
|
3436
|
+
const path = generateHumanMousePath(
|
3437
|
+
this.currentMousePosition,
|
3438
|
+
{ x: toX, y: toY },
|
3439
|
+
moveOptions
|
3440
|
+
);
|
3441
|
+
for (let i = 0; i < path.length; i++) {
|
3442
|
+
const point = path[i];
|
3443
|
+
if (moveOptions.showVisualFeedback) {
|
3444
|
+
await this.showMousePointer(point.x, point.y);
|
3445
|
+
}
|
3446
|
+
await this.underlyingPage.mouse.move(point.x, point.y);
|
3447
|
+
this.currentMousePosition = point;
|
3448
|
+
if (i < path.length - 1) {
|
3449
|
+
const delay = calculateStepDelay(
|
3450
|
+
moveOptions.duration || 300,
|
3451
|
+
i,
|
3452
|
+
path.length
|
3453
|
+
);
|
3454
|
+
await (0, import_utils15.sleep)(delay);
|
3455
|
+
}
|
3456
|
+
}
|
3457
|
+
}
|
3110
3458
|
async waitForNavigation() {
|
3111
3459
|
if (this.pageType === "puppeteer" || this.pageType === "playwright") {
|
3112
3460
|
debugPage("waitForNavigation begin");
|
@@ -3196,7 +3544,8 @@ var Page = class {
|
|
3196
3544
|
return {
|
3197
3545
|
click: async (x, y, options) => {
|
3198
3546
|
await this.mouse.move(x, y);
|
3199
|
-
this.
|
3547
|
+
await this.showClickAnimation(x, y);
|
3548
|
+
await this.underlyingPage.mouse.click(x, y, {
|
3200
3549
|
button: options?.button || "left",
|
3201
3550
|
count: options?.count || 1
|
3202
3551
|
});
|
@@ -3216,27 +3565,23 @@ var Page = class {
|
|
3216
3565
|
},
|
3217
3566
|
move: async (x, y) => {
|
3218
3567
|
this.everMoved = true;
|
3219
|
-
|
3568
|
+
await this.moveMouseHumanLike(x, y);
|
3569
|
+
return Promise.resolve();
|
3220
3570
|
},
|
3221
3571
|
drag: async (from, to) => {
|
3222
3572
|
if (this.pageType === "puppeteer") {
|
3223
|
-
await this.
|
3224
|
-
|
3225
|
-
|
3226
|
-
|
3227
|
-
|
3228
|
-
|
3229
|
-
|
3230
|
-
|
3231
|
-
}
|
3232
|
-
);
|
3573
|
+
await this.mouse.move(from.x, from.y);
|
3574
|
+
await this.underlyingPage.mouse.down();
|
3575
|
+
await this.moveMouseHumanLike(to.x, to.y, {
|
3576
|
+
duration: 500,
|
3577
|
+
// Slower for drag operations
|
3578
|
+
steps: 25
|
3579
|
+
});
|
3580
|
+
await this.underlyingPage.mouse.up();
|
3233
3581
|
} else if (this.pageType === "playwright") {
|
3234
|
-
await this.
|
3235
|
-
from.x,
|
3236
|
-
from.y
|
3237
|
-
);
|
3582
|
+
await this.mouse.move(from.x, from.y);
|
3238
3583
|
await this.underlyingPage.mouse.down();
|
3239
|
-
await this.
|
3584
|
+
await this.mouse.move(to.x, to.y);
|
3240
3585
|
await this.underlyingPage.mouse.up();
|
3241
3586
|
}
|
3242
3587
|
}
|
@@ -3299,6 +3644,18 @@ var Page = class {
|
|
3299
3644
|
await this.mouse.move(targetX, targetY);
|
3300
3645
|
}
|
3301
3646
|
}
|
3647
|
+
/**
|
3648
|
+
* Configure mouse movement behavior
|
3649
|
+
*/
|
3650
|
+
configureMouseMovement(options) {
|
3651
|
+
this.mouseMovementOptions = { ...this.mouseMovementOptions, ...options };
|
3652
|
+
}
|
3653
|
+
/**
|
3654
|
+
* Configure visual feedback
|
3655
|
+
*/
|
3656
|
+
configureVisualFeedback(options) {
|
3657
|
+
this.visualFeedbackOptions = { ...this.visualFeedbackOptions, ...options };
|
3658
|
+
}
|
3302
3659
|
async scrollUntilTop(startingPoint) {
|
3303
3660
|
await this.moveToPointBeforeScroll(startingPoint);
|
3304
3661
|
return this.mouse.wheel(0, -9999999);
|
@@ -3361,8 +3718,8 @@ var WebPage = class extends Page {
|
|
3361
3718
|
|
3362
3719
|
// src/playwright/ai-fixture.ts
|
3363
3720
|
var import_node_crypto = require("crypto");
|
3364
|
-
var import_logger6 = require("misoai-shared/logger");
|
3365
3721
|
var import_test = require("@playwright/test");
|
3722
|
+
var import_logger6 = require("misoai-shared/logger");
|
3366
3723
|
var debugPage2 = (0, import_logger6.getDebug)("web:playwright:ai-fixture");
|
3367
3724
|
var groupAndCaseForTest = (testInfo) => {
|
3368
3725
|
let taskFile;
|