misoai-web 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +108 -50
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +110 -52
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +109 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +425 -68
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +111 -53
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js +4 -4
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +108 -50
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +425 -68
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +424 -67
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +424 -67
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/utils.js +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +108 -50
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +110 -52
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +109 -51
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +425 -68
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +111 -53
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js +4 -4
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +108 -50
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +425 -68
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +424 -67
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +424 -67
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +1 -1
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-9b472ffb.d.ts → browser-f205f69d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +2 -2
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-ed0ecb44.d.ts → page-c5452809.d.ts} +45 -0
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +2 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +2 -2
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +18 -54
package/dist/lib/index.js
CHANGED
@@ -349,8 +349,8 @@ var ScriptPlayer = class {
|
|
349
349
|
var import_js_yaml = __toESM(require("js-yaml"));
|
350
350
|
|
351
351
|
// src/yaml/utils.ts
|
352
|
-
var import_utils2 = require("misoai-shared/utils");
|
353
352
|
var import_js_yaml2 = __toESM(require("js-yaml"));
|
353
|
+
var import_utils2 = require("misoai-shared/utils");
|
354
354
|
function interpolateEnvVars(content) {
|
355
355
|
return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
|
356
356
|
const value = process.env[envVar.trim()];
|
@@ -496,13 +496,13 @@ function paramStr(task) {
|
|
496
496
|
}
|
497
497
|
|
498
498
|
// src/common/utils.ts
|
499
|
+
var import_dayjs = __toESM(require("dayjs"));
|
499
500
|
var import_ai_model = require("misoai-core/ai-model");
|
500
501
|
var import_utils3 = require("misoai-core/utils");
|
501
502
|
var import_env = require("misoai-shared/env");
|
502
503
|
var import_extractor = require("misoai-shared/extractor");
|
503
504
|
var import_img = require("misoai-shared/img");
|
504
505
|
var import_utils4 = require("misoai-shared/utils");
|
505
|
-
var import_dayjs = __toESM(require("dayjs"));
|
506
506
|
|
507
507
|
// src/web-element.ts
|
508
508
|
var WebElementInfo = class {
|
@@ -671,8 +671,12 @@ var WorkflowMemory = class {
|
|
671
671
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
672
672
|
workflow.memory = [...memory];
|
673
673
|
workflow.metadata.totalSteps = workflow.steps.length;
|
674
|
-
workflow.metadata.completedSteps = workflow.steps.filter(
|
675
|
-
|
674
|
+
workflow.metadata.completedSteps = workflow.steps.filter(
|
675
|
+
(s) => s.status === "completed"
|
676
|
+
).length;
|
677
|
+
workflow.metadata.failedSteps = workflow.steps.filter(
|
678
|
+
(s) => s.status === "failed"
|
679
|
+
).length;
|
676
680
|
this.workflows.set(workflowId, workflow);
|
677
681
|
this.enforceRetentionPolicy();
|
678
682
|
}
|
@@ -683,7 +687,9 @@ var WorkflowMemory = class {
|
|
683
687
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
684
688
|
workflow.context = { ...workflow.context, ...context };
|
685
689
|
if (context.currentStep) {
|
686
|
-
const existingStep = workflow.steps.find(
|
690
|
+
const existingStep = workflow.steps.find(
|
691
|
+
(s) => s.stepName === context.currentStep
|
692
|
+
);
|
687
693
|
if (!existingStep) {
|
688
694
|
workflow.steps.push({
|
689
695
|
stepId: `step_${workflow.steps.length + 1}`,
|
@@ -728,7 +734,9 @@ var WorkflowMemory = class {
|
|
728
734
|
enforceRetentionPolicy() {
|
729
735
|
const maxWorkflows = 10;
|
730
736
|
if (this.workflows.size > maxWorkflows) {
|
731
|
-
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
737
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
738
|
+
([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
|
739
|
+
);
|
732
740
|
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
733
741
|
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
734
742
|
}
|
@@ -1295,6 +1303,7 @@ var PageTaskExecutor = class {
|
|
1295
1303
|
actions: [],
|
1296
1304
|
more_actions_needed_by_instruction: false,
|
1297
1305
|
log: "",
|
1306
|
+
summary: "Loaded YAML workflow configuration",
|
1298
1307
|
yamlString
|
1299
1308
|
},
|
1300
1309
|
cache: {
|
@@ -1396,6 +1405,7 @@ var PageTaskExecutor = class {
|
|
1396
1405
|
actions: finalActions,
|
1397
1406
|
more_actions_needed_by_instruction,
|
1398
1407
|
log: log2,
|
1408
|
+
summary: planResult.summary || "Generated action plan from user instruction",
|
1399
1409
|
yamlFlow: planResult.yamlFlow
|
1400
1410
|
},
|
1401
1411
|
cache: {
|
@@ -1451,6 +1461,7 @@ var PageTaskExecutor = class {
|
|
1451
1461
|
actionType: actions[0].type,
|
1452
1462
|
more_actions_needed_by_instruction: true,
|
1453
1463
|
log: "",
|
1464
|
+
summary: action_summary || "Generated VLM action plan",
|
1454
1465
|
yamlFlow: planResult.yamlFlow
|
1455
1466
|
},
|
1456
1467
|
cache: {
|
@@ -1467,7 +1478,9 @@ var PageTaskExecutor = class {
|
|
1467
1478
|
*/
|
1468
1479
|
getPersistentExecutor() {
|
1469
1480
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1470
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1481
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1482
|
+
this.sessionContext.workflowId
|
1483
|
+
);
|
1471
1484
|
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1472
1485
|
onTaskStart: this.onTaskStartCallback,
|
1473
1486
|
initialMemory: previousMemory
|
@@ -1496,7 +1509,9 @@ var PageTaskExecutor = class {
|
|
1496
1509
|
if (this.persistentExecutor) {
|
1497
1510
|
this.persistentExecutor.clearMemory();
|
1498
1511
|
}
|
1499
|
-
this.workflowMemory.clearWorkflow(
|
1512
|
+
this.workflowMemory.clearWorkflow(
|
1513
|
+
this.sessionContext.workflowId || "default"
|
1514
|
+
);
|
1500
1515
|
}
|
1501
1516
|
/**
|
1502
1517
|
* Mevcut hafızayı döndürür
|
@@ -1508,7 +1523,9 @@ var PageTaskExecutor = class {
|
|
1508
1523
|
* İş akışı hafızasını döndürür
|
1509
1524
|
*/
|
1510
1525
|
getWorkflowMemory() {
|
1511
|
-
return this.workflowMemory.getWorkflowData(
|
1526
|
+
return this.workflowMemory.getWorkflowData(
|
1527
|
+
this.sessionContext.workflowId || "default"
|
1528
|
+
);
|
1512
1529
|
}
|
1513
1530
|
/**
|
1514
1531
|
* Hafıza istatistiklerini döndürür
|
@@ -1516,7 +1533,13 @@ var PageTaskExecutor = class {
|
|
1516
1533
|
getMemoryStats() {
|
1517
1534
|
return this.persistentExecutor?.getMemoryStats() || {
|
1518
1535
|
totalItems: 0,
|
1519
|
-
analytics: {
|
1536
|
+
analytics: {
|
1537
|
+
totalTasks: 0,
|
1538
|
+
memoryHits: 0,
|
1539
|
+
memoryMisses: 0,
|
1540
|
+
averageMemorySize: 0,
|
1541
|
+
memoryEffectiveness: 0
|
1542
|
+
},
|
1520
1543
|
config: this.memoryConfig
|
1521
1544
|
};
|
1522
1545
|
}
|
@@ -1552,11 +1575,14 @@ var PageTaskExecutor = class {
|
|
1552
1575
|
let taskExecutor;
|
1553
1576
|
if (useMemory) {
|
1554
1577
|
taskExecutor = this.getPersistentExecutor();
|
1555
|
-
this.workflowMemory.updateWorkflowContext(
|
1556
|
-
|
1557
|
-
|
1558
|
-
|
1559
|
-
|
1578
|
+
this.workflowMemory.updateWorkflowContext(
|
1579
|
+
{
|
1580
|
+
currentStep: title,
|
1581
|
+
pageInfo: this.sessionContext.pageInfo,
|
1582
|
+
timestamp: Date.now()
|
1583
|
+
},
|
1584
|
+
this.sessionContext.workflowId || "default"
|
1585
|
+
);
|
1560
1586
|
} else {
|
1561
1587
|
taskExecutor = new import_misoai_core.Executor(title, {
|
1562
1588
|
onTaskStart: this.onTaskStartCallback
|
@@ -1922,14 +1948,19 @@ var PageTaskExecutor = class {
|
|
1922
1948
|
*/
|
1923
1949
|
addToMemory(memoryItem) {
|
1924
1950
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1925
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1951
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1952
|
+
this.sessionContext.workflowId
|
1953
|
+
);
|
1926
1954
|
this.persistentExecutor = new import_misoai_core.Executor("Persistent Task Executor", {
|
1927
1955
|
onTaskStart: this.onTaskStartCallback,
|
1928
1956
|
initialMemory: previousMemory
|
1929
1957
|
});
|
1930
1958
|
}
|
1931
1959
|
this.persistentExecutor.memoryStore?.add(memoryItem);
|
1932
|
-
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1960
|
+
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1961
|
+
"add",
|
1962
|
+
memoryItem
|
1963
|
+
);
|
1933
1964
|
}
|
1934
1965
|
};
|
1935
1966
|
|
@@ -2018,14 +2049,14 @@ function buildPlans(type, locateParam, param) {
|
|
2018
2049
|
var import_node_assert = __toESM(require("assert"));
|
2019
2050
|
var import_node_fs2 = require("fs");
|
2020
2051
|
var import_node_path2 = require("path");
|
2052
|
+
var import_js_yaml3 = __toESM(require("js-yaml"));
|
2021
2053
|
var import_common2 = require("misoai-shared/common");
|
2022
2054
|
var import_logger3 = require("misoai-shared/logger");
|
2023
2055
|
var import_utils9 = require("misoai-shared/utils");
|
2024
|
-
var import_js_yaml3 = __toESM(require("js-yaml"));
|
2025
2056
|
var import_semver = __toESM(require("semver"));
|
2026
2057
|
|
2027
2058
|
// package.json
|
2028
|
-
var version = "1.6.
|
2059
|
+
var version = "1.6.2";
|
2029
2060
|
|
2030
2061
|
// src/common/task-cache.ts
|
2031
2062
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -2320,22 +2351,28 @@ var PageAgent = class {
|
|
2320
2351
|
const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
|
2321
2352
|
const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
|
2322
2353
|
const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
|
2323
|
-
const planningTasks = executor.tasks.filter(
|
2324
|
-
|
2354
|
+
const planningTasks = executor.tasks.filter(
|
2355
|
+
(task) => task.type === "Planning"
|
2356
|
+
);
|
2357
|
+
const insightTasks = executor.tasks.filter(
|
2358
|
+
(task) => task.type === "Insight"
|
2359
|
+
);
|
2325
2360
|
const actionTasks = executor.tasks.filter((task) => task.type === "Action");
|
2326
2361
|
const planning = planningTasks.length > 0 ? {
|
2327
2362
|
type: "Planning",
|
2328
|
-
description:
|
2363
|
+
description: "Planning for task execution",
|
2329
2364
|
steps: planningTasks.map((task) => task.thought || "Planning step")
|
2330
2365
|
} : void 0;
|
2331
2366
|
const insight = insightTasks.length > 0 ? {
|
2332
2367
|
type: "Insight",
|
2333
|
-
description:
|
2334
|
-
elements: insightTasks.map(
|
2368
|
+
description: "Insight for task execution",
|
2369
|
+
elements: insightTasks.map(
|
2370
|
+
(task) => task.thought || "Insight element"
|
2371
|
+
)
|
2335
2372
|
} : void 0;
|
2336
2373
|
const action = actionTasks.length > 0 ? {
|
2337
2374
|
type: "Action",
|
2338
|
-
description:
|
2375
|
+
description: "Action for task execution",
|
2339
2376
|
result: lastTask?.output
|
2340
2377
|
} : void 0;
|
2341
2378
|
const actionDetails = executor.tasks.map((task) => ({
|
@@ -2669,7 +2706,10 @@ ${memoryContext}` : void 0;
|
|
2669
2706
|
}
|
2670
2707
|
const memoryContext = this.getMemoryAsContext();
|
2671
2708
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2672
|
-
const { output, executor } = await this.taskExecutor.assert(
|
2709
|
+
const { output, executor } = await this.taskExecutor.assert(
|
2710
|
+
assertionWithContext,
|
2711
|
+
memoryContext
|
2712
|
+
);
|
2673
2713
|
const metadata = this.afterTaskRunning(executor, true);
|
2674
2714
|
if (output && opt?.keepRawResponse) {
|
2675
2715
|
return {
|
@@ -2708,7 +2748,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
|
|
2708
2748
|
Return only "complex" or "simple" based on your analysis.
|
2709
2749
|
`;
|
2710
2750
|
const complexityMsgs = [
|
2711
|
-
{
|
2751
|
+
{
|
2752
|
+
role: "system",
|
2753
|
+
content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
|
2754
|
+
},
|
2712
2755
|
{
|
2713
2756
|
role: "user",
|
2714
2757
|
content: [
|
@@ -2732,7 +2775,12 @@ Return only "complex" or "simple" based on your analysis.
|
|
2732
2775
|
);
|
2733
2776
|
const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
|
2734
2777
|
shouldUseDeepThink = responseText.includes("complex");
|
2735
|
-
debug4(
|
2778
|
+
debug4(
|
2779
|
+
"CAPTCHA complexity analysis:",
|
2780
|
+
responseText,
|
2781
|
+
"Using deep think:",
|
2782
|
+
shouldUseDeepThink
|
2783
|
+
);
|
2736
2784
|
} catch (error) {
|
2737
2785
|
debug4("Failed to analyze CAPTCHA complexity:", error);
|
2738
2786
|
}
|
@@ -2749,7 +2797,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2749
2797
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2750
2798
|
} else if (action.type === "input" && action.value) {
|
2751
2799
|
if (action.target) {
|
2752
|
-
await this.aiInput(action.value, action.target, {
|
2800
|
+
await this.aiInput(action.value, action.target, {
|
2801
|
+
deepThink: shouldUseDeepThink
|
2802
|
+
});
|
2753
2803
|
}
|
2754
2804
|
} else if (action.type === "verify" && action.target) {
|
2755
2805
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
@@ -2761,7 +2811,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2761
2811
|
if (action.coordinates) {
|
2762
2812
|
const x = action.coordinates[0];
|
2763
2813
|
const y = action.coordinates[1];
|
2764
|
-
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2814
|
+
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2815
|
+
deepThink: shouldUseDeepThink
|
2816
|
+
});
|
2765
2817
|
} else if (action.target) {
|
2766
2818
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2767
2819
|
}
|
@@ -2913,25 +2965,27 @@ ${errors}`);
|
|
2913
2965
|
const executionDump = {
|
2914
2966
|
name: screenshotTitle,
|
2915
2967
|
description: content,
|
2916
|
-
tasks: [
|
2917
|
-
|
2918
|
-
|
2919
|
-
|
2920
|
-
|
2921
|
-
|
2922
|
-
|
2923
|
-
|
2924
|
-
|
2925
|
-
|
2926
|
-
|
2927
|
-
|
2928
|
-
|
2929
|
-
|
2930
|
-
|
2931
|
-
|
2932
|
-
|
2968
|
+
tasks: [
|
2969
|
+
{
|
2970
|
+
type: "Screenshot",
|
2971
|
+
subType: "log",
|
2972
|
+
status: "finished",
|
2973
|
+
executor: null,
|
2974
|
+
param: {
|
2975
|
+
title: screenshotTitle,
|
2976
|
+
content
|
2977
|
+
},
|
2978
|
+
output: {
|
2979
|
+
screenshot
|
2980
|
+
},
|
2981
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2982
|
+
timing: {
|
2983
|
+
start: Date.now(),
|
2984
|
+
end: Date.now(),
|
2985
|
+
cost: 0
|
2986
|
+
}
|
2933
2987
|
}
|
2934
|
-
|
2988
|
+
],
|
2935
2989
|
sdkVersion: "1.0.0",
|
2936
2990
|
logTime: Date.now(),
|
2937
2991
|
model_name: "screenshot"
|
@@ -2983,7 +3037,9 @@ ${errors}`);
|
|
2983
3037
|
totalTasks: stats.analytics.totalTasks,
|
2984
3038
|
memoryHits: stats.analytics.memoryHits,
|
2985
3039
|
memoryMisses: stats.analytics.memoryMisses,
|
2986
|
-
memoryEffectiveness: Math.round(
|
3040
|
+
memoryEffectiveness: Math.round(
|
3041
|
+
stats.analytics.memoryEffectiveness * 100
|
3042
|
+
),
|
2987
3043
|
averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
|
2988
3044
|
},
|
2989
3045
|
config: stats.config,
|
@@ -3047,7 +3103,9 @@ ${errors}`);
|
|
3047
3103
|
calculateSuccessRate(memory) {
|
3048
3104
|
if (memory.length === 0)
|
3049
3105
|
return 0;
|
3050
|
-
const successCount = memory.filter(
|
3106
|
+
const successCount = memory.filter(
|
3107
|
+
(item) => item.metadata?.success !== false
|
3108
|
+
).length;
|
3051
3109
|
return Math.round(successCount / memory.length * 100);
|
3052
3110
|
}
|
3053
3111
|
calculateAverageExecutionTime(memory) {
|
@@ -3083,13 +3141,236 @@ var import_extractor2 = require("misoai-shared/extractor");
|
|
3083
3141
|
var import_fs = require("misoai-shared/fs");
|
3084
3142
|
var import_logger5 = require("misoai-shared/logger");
|
3085
3143
|
var import_utils16 = require("misoai-shared/utils");
|
3144
|
+
|
3145
|
+
// src/common/mouse-utils.ts
|
3146
|
+
function generateHumanMousePath(from, to, options = {}) {
|
3147
|
+
const {
|
3148
|
+
steps = 20,
|
3149
|
+
easing = "easeInOut"
|
3150
|
+
} = options;
|
3151
|
+
if (steps <= 1) {
|
3152
|
+
return [to];
|
3153
|
+
}
|
3154
|
+
const path = [];
|
3155
|
+
const controlPoint1 = {
|
3156
|
+
x: from.x + (to.x - from.x) * 0.25 + (Math.random() - 0.5) * 50,
|
3157
|
+
y: from.y + (to.y - from.y) * 0.25 + (Math.random() - 0.5) * 50
|
3158
|
+
};
|
3159
|
+
const controlPoint2 = {
|
3160
|
+
x: from.x + (to.x - from.x) * 0.75 + (Math.random() - 0.5) * 50,
|
3161
|
+
y: from.y + (to.y - from.y) * 0.75 + (Math.random() - 0.5) * 50
|
3162
|
+
};
|
3163
|
+
for (let i = 0; i <= steps; i++) {
|
3164
|
+
const t = i / steps;
|
3165
|
+
const easedT = applyEasing(t, easing);
|
3166
|
+
const point = calculateBezierPoint(from, controlPoint1, controlPoint2, to, easedT);
|
3167
|
+
path.push(point);
|
3168
|
+
}
|
3169
|
+
return path;
|
3170
|
+
}
|
3171
|
+
function calculateBezierPoint(p0, p1, p2, p3, t) {
|
3172
|
+
const oneMinusT = 1 - t;
|
3173
|
+
const oneMinusTSquared = oneMinusT * oneMinusT;
|
3174
|
+
const oneMinusTCubed = oneMinusTSquared * oneMinusT;
|
3175
|
+
const tSquared = t * t;
|
3176
|
+
const tCubed = tSquared * t;
|
3177
|
+
return {
|
3178
|
+
x: oneMinusTCubed * p0.x + 3 * oneMinusTSquared * t * p1.x + 3 * oneMinusT * tSquared * p2.x + tCubed * p3.x,
|
3179
|
+
y: oneMinusTCubed * p0.y + 3 * oneMinusTSquared * t * p1.y + 3 * oneMinusT * tSquared * p2.y + tCubed * p3.y
|
3180
|
+
};
|
3181
|
+
}
|
3182
|
+
function applyEasing(t, easing) {
|
3183
|
+
switch (easing) {
|
3184
|
+
case "linear":
|
3185
|
+
return t;
|
3186
|
+
case "easeIn":
|
3187
|
+
return t * t;
|
3188
|
+
case "easeOut":
|
3189
|
+
return 1 - (1 - t) * (1 - t);
|
3190
|
+
case "easeInOut":
|
3191
|
+
return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
|
3192
|
+
default:
|
3193
|
+
return t;
|
3194
|
+
}
|
3195
|
+
}
|
3196
|
+
function calculateStepDelay(totalDuration, currentStep, totalSteps) {
|
3197
|
+
const baseDelay = totalDuration / totalSteps;
|
3198
|
+
const variation = 0.3;
|
3199
|
+
const randomFactor = 1 + (Math.random() - 0.5) * variation;
|
3200
|
+
return Math.max(1, baseDelay * randomFactor);
|
3201
|
+
}
|
3202
|
+
|
3203
|
+
// src/common/visual-feedback.ts
|
3204
|
+
function getMousePointerScript(x, y, options = {}) {
|
3205
|
+
const {
|
3206
|
+
pointerSize = 20,
|
3207
|
+
pointerColor = "#ff4444",
|
3208
|
+
trailLength = 5,
|
3209
|
+
showTrail = true,
|
3210
|
+
animationDuration = 200
|
3211
|
+
} = options;
|
3212
|
+
return `
|
3213
|
+
(() => {
|
3214
|
+
// Create or update mouse pointer
|
3215
|
+
if (!window.misoaiMousePointer) {
|
3216
|
+
const pointer = document.createElement('div');
|
3217
|
+
pointer.id = 'misoai-mouse-pointer';
|
3218
|
+
pointer.style.cssText = \`
|
3219
|
+
position: fixed;
|
3220
|
+
width: ${pointerSize}px;
|
3221
|
+
height: ${pointerSize}px;
|
3222
|
+
background: ${pointerColor};
|
3223
|
+
border: 2px solid white;
|
3224
|
+
border-radius: 50%;
|
3225
|
+
pointer-events: none;
|
3226
|
+
z-index: 999999;
|
3227
|
+
box-shadow: 0 0 10px rgba(0,0,0,0.3);
|
3228
|
+
transition: all ${animationDuration}ms ease-out;
|
3229
|
+
transform: translate(-50%, -50%);
|
3230
|
+
\`;
|
3231
|
+
document.body.appendChild(pointer);
|
3232
|
+
window.misoaiMousePointer = pointer;
|
3233
|
+
|
3234
|
+
// Initialize trail array
|
3235
|
+
window.misoaiMouseTrail = [];
|
3236
|
+
}
|
3237
|
+
|
3238
|
+
const pointer = window.misoaiMousePointer;
|
3239
|
+
pointer.style.left = ${x}px;
|
3240
|
+
pointer.style.top = ${y}px;
|
3241
|
+
|
3242
|
+
${showTrail ? getTrailScript(x, y, trailLength, pointerColor) : ""}
|
3243
|
+
|
3244
|
+
// Auto-hide after 3 seconds of inactivity
|
3245
|
+
clearTimeout(window.misoaiMouseTimeout);
|
3246
|
+
pointer.style.opacity = '1';
|
3247
|
+
window.misoaiMouseTimeout = setTimeout(() => {
|
3248
|
+
if (pointer) pointer.style.opacity = '0.3';
|
3249
|
+
}, 3000);
|
3250
|
+
})();
|
3251
|
+
`;
|
3252
|
+
}
|
3253
|
+
function getTrailScript(x, y, trailLength, color) {
|
3254
|
+
return `
|
3255
|
+
// Add current position to trail
|
3256
|
+
window.misoaiMouseTrail.push({ x: ${x}, y: ${y}, timestamp: Date.now() });
|
3257
|
+
|
3258
|
+
// Keep only recent trail points
|
3259
|
+
if (window.misoaiMouseTrail.length > ${trailLength}) {
|
3260
|
+
window.misoaiMouseTrail.shift();
|
3261
|
+
}
|
3262
|
+
|
3263
|
+
// Remove old trail elements
|
3264
|
+
document.querySelectorAll('.misoai-mouse-trail').forEach(el => el.remove());
|
3265
|
+
|
3266
|
+
// Create new trail elements
|
3267
|
+
window.misoaiMouseTrail.forEach((point, index) => {
|
3268
|
+
if (index === window.misoaiMouseTrail.length - 1) return; // Skip current position
|
3269
|
+
|
3270
|
+
const trailElement = document.createElement('div');
|
3271
|
+
trailElement.className = 'misoai-mouse-trail';
|
3272
|
+
const opacity = (index + 1) / window.misoaiMouseTrail.length * 0.6;
|
3273
|
+
const size = 8 + (index + 1) / window.misoaiMouseTrail.length * 8;
|
3274
|
+
|
3275
|
+
trailElement.style.cssText = \`
|
3276
|
+
position: fixed;
|
3277
|
+
width: \${size}px;
|
3278
|
+
height: \${size}px;
|
3279
|
+
background: ${color};
|
3280
|
+
border-radius: 50%;
|
3281
|
+
pointer-events: none;
|
3282
|
+
z-index: 999998;
|
3283
|
+
opacity: \${opacity};
|
3284
|
+
transform: translate(-50%, -50%);
|
3285
|
+
left: \${point.x}px;
|
3286
|
+
top: \${point.y}px;
|
3287
|
+
\`;
|
3288
|
+
|
3289
|
+
document.body.appendChild(trailElement);
|
3290
|
+
|
3291
|
+
// Auto-remove trail element after animation
|
3292
|
+
setTimeout(() => {
|
3293
|
+
if (trailElement.parentNode) {
|
3294
|
+
trailElement.parentNode.removeChild(trailElement);
|
3295
|
+
}
|
3296
|
+
}, 1000);
|
3297
|
+
});
|
3298
|
+
`;
|
3299
|
+
}
|
3300
|
+
function getClickAnimationScript(x, y) {
|
3301
|
+
return `
|
3302
|
+
(() => {
|
3303
|
+
const clickRipple = document.createElement('div');
|
3304
|
+
clickRipple.style.cssText = \`
|
3305
|
+
position: fixed;
|
3306
|
+
left: ${x}px;
|
3307
|
+
top: ${y}px;
|
3308
|
+
width: 0;
|
3309
|
+
height: 0;
|
3310
|
+
border: 2px solid #ff4444;
|
3311
|
+
border-radius: 50%;
|
3312
|
+
pointer-events: none;
|
3313
|
+
z-index: 999999;
|
3314
|
+
transform: translate(-50%, -50%);
|
3315
|
+
animation: misoai-click-ripple 0.6s ease-out forwards;
|
3316
|
+
\`;
|
3317
|
+
|
3318
|
+
// Add CSS animation if not exists
|
3319
|
+
if (!document.getElementById('misoai-click-styles')) {
|
3320
|
+
const style = document.createElement('style');
|
3321
|
+
style.id = 'misoai-click-styles';
|
3322
|
+
style.textContent = \`
|
3323
|
+
@keyframes misoai-click-ripple {
|
3324
|
+
0% {
|
3325
|
+
width: 0;
|
3326
|
+
height: 0;
|
3327
|
+
opacity: 1;
|
3328
|
+
}
|
3329
|
+
100% {
|
3330
|
+
width: 40px;
|
3331
|
+
height: 40px;
|
3332
|
+
opacity: 0;
|
3333
|
+
}
|
3334
|
+
}
|
3335
|
+
\`;
|
3336
|
+
document.head.appendChild(style);
|
3337
|
+
}
|
3338
|
+
|
3339
|
+
document.body.appendChild(clickRipple);
|
3340
|
+
|
3341
|
+
// Remove after animation
|
3342
|
+
setTimeout(() => {
|
3343
|
+
if (clickRipple.parentNode) {
|
3344
|
+
clickRipple.parentNode.removeChild(clickRipple);
|
3345
|
+
}
|
3346
|
+
}, 600);
|
3347
|
+
})();
|
3348
|
+
`;
|
3349
|
+
}
|
3350
|
+
|
3351
|
+
// src/puppeteer/base-page.ts
|
3086
3352
|
var debugPage = (0, import_logger5.getDebug)("web:page");
|
3087
3353
|
var Page = class {
|
3088
3354
|
constructor(underlyingPage, pageType, opts) {
|
3355
|
+
this.currentMousePosition = { x: 0, y: 0 };
|
3356
|
+
this.mouseMovementOptions = {
|
3357
|
+
steps: 15,
|
3358
|
+
duration: 300,
|
3359
|
+
easing: "easeInOut",
|
3360
|
+
showVisualFeedback: true
|
3361
|
+
};
|
3362
|
+
this.visualFeedbackOptions = {
|
3363
|
+
pointerSize: 16,
|
3364
|
+
pointerColor: "#ff4444",
|
3365
|
+
trailLength: 4,
|
3366
|
+
showTrail: true,
|
3367
|
+
animationDuration: 150
|
3368
|
+
};
|
3089
3369
|
this.everMoved = false;
|
3090
3370
|
this.underlyingPage = underlyingPage;
|
3091
3371
|
this.pageType = pageType;
|
3092
3372
|
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
3373
|
+
this.initializeMousePosition();
|
3093
3374
|
}
|
3094
3375
|
async evaluate(pageFunction, arg) {
|
3095
3376
|
let result;
|
@@ -3108,9 +3389,76 @@ var Page = class {
|
|
3108
3389
|
debugPage("evaluate function end");
|
3109
3390
|
return result;
|
3110
3391
|
}
|
3392
|
+
/**
|
3393
|
+
* Initialize mouse position to center of viewport
|
3394
|
+
*/
|
3395
|
+
async initializeMousePosition() {
|
3396
|
+
try {
|
3397
|
+
const size = await this.size();
|
3398
|
+
this.currentMousePosition = {
|
3399
|
+
x: Math.floor(size.width / 2),
|
3400
|
+
y: Math.floor(size.height / 2)
|
3401
|
+
};
|
3402
|
+
} catch (error) {
|
3403
|
+
this.currentMousePosition = { x: 400, y: 300 };
|
3404
|
+
}
|
3405
|
+
}
|
3111
3406
|
async evaluateJavaScript(script) {
|
3112
3407
|
return this.evaluate(script);
|
3113
3408
|
}
|
3409
|
+
/**
|
3410
|
+
* Show visual feedback for mouse pointer
|
3411
|
+
*/
|
3412
|
+
async showMousePointer(x, y) {
|
3413
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3414
|
+
return;
|
3415
|
+
const script = getMousePointerScript(x, y, this.visualFeedbackOptions);
|
3416
|
+
try {
|
3417
|
+
await this.evaluate(script);
|
3418
|
+
} catch (error) {
|
3419
|
+
debugPage("Visual feedback error:", error);
|
3420
|
+
}
|
3421
|
+
}
|
3422
|
+
/**
|
3423
|
+
* Show click animation
|
3424
|
+
*/
|
3425
|
+
async showClickAnimation(x, y) {
|
3426
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3427
|
+
return;
|
3428
|
+
const script = getClickAnimationScript(x, y);
|
3429
|
+
try {
|
3430
|
+
await this.evaluate(script);
|
3431
|
+
} catch (error) {
|
3432
|
+
debugPage("Click animation error:", error);
|
3433
|
+
}
|
3434
|
+
}
|
3435
|
+
/**
|
3436
|
+
* Move mouse with human-like movement and visual feedback
|
3437
|
+
*/
|
3438
|
+
async moveMouseHumanLike(toX, toY, options) {
|
3439
|
+
const moveOptions = { ...this.mouseMovementOptions, ...options };
|
3440
|
+
const path = generateHumanMousePath(
|
3441
|
+
this.currentMousePosition,
|
3442
|
+
{ x: toX, y: toY },
|
3443
|
+
moveOptions
|
3444
|
+
);
|
3445
|
+
for (let i = 0; i < path.length; i++) {
|
3446
|
+
const point = path[i];
|
3447
|
+
if (moveOptions.showVisualFeedback) {
|
3448
|
+
await this.showMousePointer(point.x, point.y);
|
3449
|
+
}
|
3450
|
+
await this.underlyingPage.mouse.move(point.x, point.y);
|
3451
|
+
this.currentMousePosition = point;
|
3452
|
+
if (i < path.length - 1) {
|
3453
|
+
const delay = calculateStepDelay(
|
3454
|
+
moveOptions.duration || 300,
|
3455
|
+
i,
|
3456
|
+
path.length
|
3457
|
+
);
|
3458
|
+
await (0, import_utils15.sleep)(delay);
|
3459
|
+
}
|
3460
|
+
}
|
3461
|
+
}
|
3114
3462
|
async waitForNavigation() {
|
3115
3463
|
if (this.pageType === "puppeteer" || this.pageType === "playwright") {
|
3116
3464
|
debugPage("waitForNavigation begin");
|
@@ -3200,7 +3548,8 @@ var Page = class {
|
|
3200
3548
|
return {
|
3201
3549
|
click: async (x, y, options) => {
|
3202
3550
|
await this.mouse.move(x, y);
|
3203
|
-
this.
|
3551
|
+
await this.showClickAnimation(x, y);
|
3552
|
+
await this.underlyingPage.mouse.click(x, y, {
|
3204
3553
|
button: options?.button || "left",
|
3205
3554
|
count: options?.count || 1
|
3206
3555
|
});
|
@@ -3220,27 +3569,23 @@ var Page = class {
|
|
3220
3569
|
},
|
3221
3570
|
move: async (x, y) => {
|
3222
3571
|
this.everMoved = true;
|
3223
|
-
|
3572
|
+
await this.moveMouseHumanLike(x, y);
|
3573
|
+
return Promise.resolve();
|
3224
3574
|
},
|
3225
3575
|
drag: async (from, to) => {
|
3226
3576
|
if (this.pageType === "puppeteer") {
|
3227
|
-
await this.
|
3228
|
-
|
3229
|
-
|
3230
|
-
|
3231
|
-
|
3232
|
-
|
3233
|
-
|
3234
|
-
|
3235
|
-
}
|
3236
|
-
);
|
3577
|
+
await this.mouse.move(from.x, from.y);
|
3578
|
+
await this.underlyingPage.mouse.down();
|
3579
|
+
await this.moveMouseHumanLike(to.x, to.y, {
|
3580
|
+
duration: 500,
|
3581
|
+
// Slower for drag operations
|
3582
|
+
steps: 25
|
3583
|
+
});
|
3584
|
+
await this.underlyingPage.mouse.up();
|
3237
3585
|
} else if (this.pageType === "playwright") {
|
3238
|
-
await this.
|
3239
|
-
from.x,
|
3240
|
-
from.y
|
3241
|
-
);
|
3586
|
+
await this.mouse.move(from.x, from.y);
|
3242
3587
|
await this.underlyingPage.mouse.down();
|
3243
|
-
await this.
|
3588
|
+
await this.mouse.move(to.x, to.y);
|
3244
3589
|
await this.underlyingPage.mouse.up();
|
3245
3590
|
}
|
3246
3591
|
}
|
@@ -3303,6 +3648,18 @@ var Page = class {
|
|
3303
3648
|
await this.mouse.move(targetX, targetY);
|
3304
3649
|
}
|
3305
3650
|
}
|
3651
|
+
/**
|
3652
|
+
* Configure mouse movement behavior
|
3653
|
+
*/
|
3654
|
+
configureMouseMovement(options) {
|
3655
|
+
this.mouseMovementOptions = { ...this.mouseMovementOptions, ...options };
|
3656
|
+
}
|
3657
|
+
/**
|
3658
|
+
* Configure visual feedback
|
3659
|
+
*/
|
3660
|
+
configureVisualFeedback(options) {
|
3661
|
+
this.visualFeedbackOptions = { ...this.visualFeedbackOptions, ...options };
|
3662
|
+
}
|
3306
3663
|
async scrollUntilTop(startingPoint) {
|
3307
3664
|
await this.moveToPointBeforeScroll(startingPoint);
|
3308
3665
|
return this.mouse.wheel(0, -9999999);
|
@@ -3365,8 +3722,8 @@ var WebPage = class extends Page {
|
|
3365
3722
|
|
3366
3723
|
// src/playwright/ai-fixture.ts
|
3367
3724
|
var import_node_crypto = require("crypto");
|
3368
|
-
var import_logger6 = require("misoai-shared/logger");
|
3369
3725
|
var import_test = require("@playwright/test");
|
3726
|
+
var import_logger6 = require("misoai-shared/logger");
|
3370
3727
|
var debugPage2 = (0, import_logger6.getDebug)("web:playwright:ai-fixture");
|
3371
3728
|
var groupAndCaseForTest = (testInfo) => {
|
3372
3729
|
let taskFile;
|