misoai-web 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +108 -50
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +110 -52
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +109 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +425 -68
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +111 -53
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js +4 -4
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +108 -50
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +425 -68
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +424 -67
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +424 -67
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/utils.js +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +108 -50
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +110 -52
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +109 -51
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +425 -68
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +111 -53
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js +4 -4
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +108 -50
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +425 -68
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +424 -67
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +424 -67
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +1 -1
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-9b472ffb.d.ts → browser-f205f69d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +2 -2
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-ed0ecb44.d.ts → page-c5452809.d.ts} +45 -0
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +2 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +2 -2
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +18 -54
package/dist/es/puppeteer.js
CHANGED
@@ -306,8 +306,8 @@ var ScriptPlayer = class {
|
|
306
306
|
import yaml from "js-yaml";
|
307
307
|
|
308
308
|
// src/yaml/utils.ts
|
309
|
-
import { assert as assert2 } from "misoai-shared/utils";
|
310
309
|
import yaml2 from "js-yaml";
|
310
|
+
import { assert as assert2 } from "misoai-shared/utils";
|
311
311
|
function interpolateEnvVars(content) {
|
312
312
|
return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
|
313
313
|
const value = process.env[envVar.trim()];
|
@@ -468,6 +468,7 @@ function paramStr(task) {
|
|
468
468
|
}
|
469
469
|
|
470
470
|
// src/common/utils.ts
|
471
|
+
import dayjs from "dayjs";
|
471
472
|
import { elementByPositionWithElementInfo } from "misoai-core/ai-model";
|
472
473
|
import { uploadTestInfoToServer } from "misoai-core/utils";
|
473
474
|
import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "misoai-shared/env";
|
@@ -479,7 +480,6 @@ import {
|
|
479
480
|
} from "misoai-shared/extractor";
|
480
481
|
import { resizeImgBase64 } from "misoai-shared/img";
|
481
482
|
import { assert as assert3, logMsg, uuid } from "misoai-shared/utils";
|
482
|
-
import dayjs from "dayjs";
|
483
483
|
|
484
484
|
// src/web-element.ts
|
485
485
|
var WebElementInfo = class {
|
@@ -648,8 +648,12 @@ var WorkflowMemory = class {
|
|
648
648
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
649
649
|
workflow.memory = [...memory];
|
650
650
|
workflow.metadata.totalSteps = workflow.steps.length;
|
651
|
-
workflow.metadata.completedSteps = workflow.steps.filter(
|
652
|
-
|
651
|
+
workflow.metadata.completedSteps = workflow.steps.filter(
|
652
|
+
(s) => s.status === "completed"
|
653
|
+
).length;
|
654
|
+
workflow.metadata.failedSteps = workflow.steps.filter(
|
655
|
+
(s) => s.status === "failed"
|
656
|
+
).length;
|
653
657
|
this.workflows.set(workflowId, workflow);
|
654
658
|
this.enforceRetentionPolicy();
|
655
659
|
}
|
@@ -660,7 +664,9 @@ var WorkflowMemory = class {
|
|
660
664
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
661
665
|
workflow.context = { ...workflow.context, ...context };
|
662
666
|
if (context.currentStep) {
|
663
|
-
const existingStep = workflow.steps.find(
|
667
|
+
const existingStep = workflow.steps.find(
|
668
|
+
(s) => s.stepName === context.currentStep
|
669
|
+
);
|
664
670
|
if (!existingStep) {
|
665
671
|
workflow.steps.push({
|
666
672
|
stepId: `step_${workflow.steps.length + 1}`,
|
@@ -705,7 +711,9 @@ var WorkflowMemory = class {
|
|
705
711
|
enforceRetentionPolicy() {
|
706
712
|
const maxWorkflows = 10;
|
707
713
|
if (this.workflows.size > maxWorkflows) {
|
708
|
-
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
714
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
715
|
+
([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
|
716
|
+
);
|
709
717
|
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
710
718
|
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
711
719
|
}
|
@@ -1272,6 +1280,7 @@ var PageTaskExecutor = class {
|
|
1272
1280
|
actions: [],
|
1273
1281
|
more_actions_needed_by_instruction: false,
|
1274
1282
|
log: "",
|
1283
|
+
summary: "Loaded YAML workflow configuration",
|
1275
1284
|
yamlString
|
1276
1285
|
},
|
1277
1286
|
cache: {
|
@@ -1373,6 +1382,7 @@ var PageTaskExecutor = class {
|
|
1373
1382
|
actions: finalActions,
|
1374
1383
|
more_actions_needed_by_instruction,
|
1375
1384
|
log: log2,
|
1385
|
+
summary: planResult.summary || "Generated action plan from user instruction",
|
1376
1386
|
yamlFlow: planResult.yamlFlow
|
1377
1387
|
},
|
1378
1388
|
cache: {
|
@@ -1428,6 +1438,7 @@ var PageTaskExecutor = class {
|
|
1428
1438
|
actionType: actions[0].type,
|
1429
1439
|
more_actions_needed_by_instruction: true,
|
1430
1440
|
log: "",
|
1441
|
+
summary: action_summary || "Generated VLM action plan",
|
1431
1442
|
yamlFlow: planResult.yamlFlow
|
1432
1443
|
},
|
1433
1444
|
cache: {
|
@@ -1444,7 +1455,9 @@ var PageTaskExecutor = class {
|
|
1444
1455
|
*/
|
1445
1456
|
getPersistentExecutor() {
|
1446
1457
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1447
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1458
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1459
|
+
this.sessionContext.workflowId
|
1460
|
+
);
|
1448
1461
|
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1449
1462
|
onTaskStart: this.onTaskStartCallback,
|
1450
1463
|
initialMemory: previousMemory
|
@@ -1473,7 +1486,9 @@ var PageTaskExecutor = class {
|
|
1473
1486
|
if (this.persistentExecutor) {
|
1474
1487
|
this.persistentExecutor.clearMemory();
|
1475
1488
|
}
|
1476
|
-
this.workflowMemory.clearWorkflow(
|
1489
|
+
this.workflowMemory.clearWorkflow(
|
1490
|
+
this.sessionContext.workflowId || "default"
|
1491
|
+
);
|
1477
1492
|
}
|
1478
1493
|
/**
|
1479
1494
|
* Mevcut hafızayı döndürür
|
@@ -1485,7 +1500,9 @@ var PageTaskExecutor = class {
|
|
1485
1500
|
* İş akışı hafızasını döndürür
|
1486
1501
|
*/
|
1487
1502
|
getWorkflowMemory() {
|
1488
|
-
return this.workflowMemory.getWorkflowData(
|
1503
|
+
return this.workflowMemory.getWorkflowData(
|
1504
|
+
this.sessionContext.workflowId || "default"
|
1505
|
+
);
|
1489
1506
|
}
|
1490
1507
|
/**
|
1491
1508
|
* Hafıza istatistiklerini döndürür
|
@@ -1493,7 +1510,13 @@ var PageTaskExecutor = class {
|
|
1493
1510
|
getMemoryStats() {
|
1494
1511
|
return this.persistentExecutor?.getMemoryStats() || {
|
1495
1512
|
totalItems: 0,
|
1496
|
-
analytics: {
|
1513
|
+
analytics: {
|
1514
|
+
totalTasks: 0,
|
1515
|
+
memoryHits: 0,
|
1516
|
+
memoryMisses: 0,
|
1517
|
+
averageMemorySize: 0,
|
1518
|
+
memoryEffectiveness: 0
|
1519
|
+
},
|
1497
1520
|
config: this.memoryConfig
|
1498
1521
|
};
|
1499
1522
|
}
|
@@ -1529,11 +1552,14 @@ var PageTaskExecutor = class {
|
|
1529
1552
|
let taskExecutor;
|
1530
1553
|
if (useMemory) {
|
1531
1554
|
taskExecutor = this.getPersistentExecutor();
|
1532
|
-
this.workflowMemory.updateWorkflowContext(
|
1533
|
-
|
1534
|
-
|
1535
|
-
|
1536
|
-
|
1555
|
+
this.workflowMemory.updateWorkflowContext(
|
1556
|
+
{
|
1557
|
+
currentStep: title,
|
1558
|
+
pageInfo: this.sessionContext.pageInfo,
|
1559
|
+
timestamp: Date.now()
|
1560
|
+
},
|
1561
|
+
this.sessionContext.workflowId || "default"
|
1562
|
+
);
|
1537
1563
|
} else {
|
1538
1564
|
taskExecutor = new Executor(title, {
|
1539
1565
|
onTaskStart: this.onTaskStartCallback
|
@@ -1899,14 +1925,19 @@ var PageTaskExecutor = class {
|
|
1899
1925
|
*/
|
1900
1926
|
addToMemory(memoryItem) {
|
1901
1927
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1902
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1928
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1929
|
+
this.sessionContext.workflowId
|
1930
|
+
);
|
1903
1931
|
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1904
1932
|
onTaskStart: this.onTaskStartCallback,
|
1905
1933
|
initialMemory: previousMemory
|
1906
1934
|
});
|
1907
1935
|
}
|
1908
1936
|
this.persistentExecutor.memoryStore?.add(memoryItem);
|
1909
|
-
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1937
|
+
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1938
|
+
"add",
|
1939
|
+
memoryItem
|
1940
|
+
);
|
1910
1941
|
}
|
1911
1942
|
};
|
1912
1943
|
|
@@ -1995,14 +2026,14 @@ function buildPlans(type, locateParam, param) {
|
|
1995
2026
|
import assert6 from "assert";
|
1996
2027
|
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
1997
2028
|
import { dirname as dirname2, join as join2 } from "path";
|
2029
|
+
import yaml3 from "js-yaml";
|
1998
2030
|
import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
|
1999
2031
|
import { getDebug as getDebug3 } from "misoai-shared/logger";
|
2000
2032
|
import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
|
2001
|
-
import yaml3 from "js-yaml";
|
2002
2033
|
import semver from "semver";
|
2003
2034
|
|
2004
2035
|
// package.json
|
2005
|
-
var version = "1.6.
|
2036
|
+
var version = "1.6.2";
|
2006
2037
|
|
2007
2038
|
// src/common/task-cache.ts
|
2008
2039
|
var debug3 = getDebug3("cache");
|
@@ -2297,22 +2328,28 @@ var PageAgent = class {
|
|
2297
2328
|
const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
|
2298
2329
|
const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
|
2299
2330
|
const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
|
2300
|
-
const planningTasks = executor.tasks.filter(
|
2301
|
-
|
2331
|
+
const planningTasks = executor.tasks.filter(
|
2332
|
+
(task) => task.type === "Planning"
|
2333
|
+
);
|
2334
|
+
const insightTasks = executor.tasks.filter(
|
2335
|
+
(task) => task.type === "Insight"
|
2336
|
+
);
|
2302
2337
|
const actionTasks = executor.tasks.filter((task) => task.type === "Action");
|
2303
2338
|
const planning = planningTasks.length > 0 ? {
|
2304
2339
|
type: "Planning",
|
2305
|
-
description:
|
2340
|
+
description: "Planning for task execution",
|
2306
2341
|
steps: planningTasks.map((task) => task.thought || "Planning step")
|
2307
2342
|
} : void 0;
|
2308
2343
|
const insight = insightTasks.length > 0 ? {
|
2309
2344
|
type: "Insight",
|
2310
|
-
description:
|
2311
|
-
elements: insightTasks.map(
|
2345
|
+
description: "Insight for task execution",
|
2346
|
+
elements: insightTasks.map(
|
2347
|
+
(task) => task.thought || "Insight element"
|
2348
|
+
)
|
2312
2349
|
} : void 0;
|
2313
2350
|
const action = actionTasks.length > 0 ? {
|
2314
2351
|
type: "Action",
|
2315
|
-
description:
|
2352
|
+
description: "Action for task execution",
|
2316
2353
|
result: lastTask?.output
|
2317
2354
|
} : void 0;
|
2318
2355
|
const actionDetails = executor.tasks.map((task) => ({
|
@@ -2646,7 +2683,10 @@ ${memoryContext}` : void 0;
|
|
2646
2683
|
}
|
2647
2684
|
const memoryContext = this.getMemoryAsContext();
|
2648
2685
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2649
|
-
const { output, executor } = await this.taskExecutor.assert(
|
2686
|
+
const { output, executor } = await this.taskExecutor.assert(
|
2687
|
+
assertionWithContext,
|
2688
|
+
memoryContext
|
2689
|
+
);
|
2650
2690
|
const metadata = this.afterTaskRunning(executor, true);
|
2651
2691
|
if (output && opt?.keepRawResponse) {
|
2652
2692
|
return {
|
@@ -2685,7 +2725,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
|
|
2685
2725
|
Return only "complex" or "simple" based on your analysis.
|
2686
2726
|
`;
|
2687
2727
|
const complexityMsgs = [
|
2688
|
-
{
|
2728
|
+
{
|
2729
|
+
role: "system",
|
2730
|
+
content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
|
2731
|
+
},
|
2689
2732
|
{
|
2690
2733
|
role: "user",
|
2691
2734
|
content: [
|
@@ -2709,7 +2752,12 @@ Return only "complex" or "simple" based on your analysis.
|
|
2709
2752
|
);
|
2710
2753
|
const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
|
2711
2754
|
shouldUseDeepThink = responseText.includes("complex");
|
2712
|
-
debug4(
|
2755
|
+
debug4(
|
2756
|
+
"CAPTCHA complexity analysis:",
|
2757
|
+
responseText,
|
2758
|
+
"Using deep think:",
|
2759
|
+
shouldUseDeepThink
|
2760
|
+
);
|
2713
2761
|
} catch (error) {
|
2714
2762
|
debug4("Failed to analyze CAPTCHA complexity:", error);
|
2715
2763
|
}
|
@@ -2726,7 +2774,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2726
2774
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2727
2775
|
} else if (action.type === "input" && action.value) {
|
2728
2776
|
if (action.target) {
|
2729
|
-
await this.aiInput(action.value, action.target, {
|
2777
|
+
await this.aiInput(action.value, action.target, {
|
2778
|
+
deepThink: shouldUseDeepThink
|
2779
|
+
});
|
2730
2780
|
}
|
2731
2781
|
} else if (action.type === "verify" && action.target) {
|
2732
2782
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
@@ -2738,7 +2788,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2738
2788
|
if (action.coordinates) {
|
2739
2789
|
const x = action.coordinates[0];
|
2740
2790
|
const y = action.coordinates[1];
|
2741
|
-
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2791
|
+
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2792
|
+
deepThink: shouldUseDeepThink
|
2793
|
+
});
|
2742
2794
|
} else if (action.target) {
|
2743
2795
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2744
2796
|
}
|
@@ -2890,25 +2942,27 @@ ${errors}`);
|
|
2890
2942
|
const executionDump = {
|
2891
2943
|
name: screenshotTitle,
|
2892
2944
|
description: content,
|
2893
|
-
tasks: [
|
2894
|
-
|
2895
|
-
|
2896
|
-
|
2897
|
-
|
2898
|
-
|
2899
|
-
|
2900
|
-
|
2901
|
-
|
2902
|
-
|
2903
|
-
|
2904
|
-
|
2905
|
-
|
2906
|
-
|
2907
|
-
|
2908
|
-
|
2909
|
-
|
2945
|
+
tasks: [
|
2946
|
+
{
|
2947
|
+
type: "Screenshot",
|
2948
|
+
subType: "log",
|
2949
|
+
status: "finished",
|
2950
|
+
executor: null,
|
2951
|
+
param: {
|
2952
|
+
title: screenshotTitle,
|
2953
|
+
content
|
2954
|
+
},
|
2955
|
+
output: {
|
2956
|
+
screenshot
|
2957
|
+
},
|
2958
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2959
|
+
timing: {
|
2960
|
+
start: Date.now(),
|
2961
|
+
end: Date.now(),
|
2962
|
+
cost: 0
|
2963
|
+
}
|
2910
2964
|
}
|
2911
|
-
|
2965
|
+
],
|
2912
2966
|
sdkVersion: "1.0.0",
|
2913
2967
|
logTime: Date.now(),
|
2914
2968
|
model_name: "screenshot"
|
@@ -2960,7 +3014,9 @@ ${errors}`);
|
|
2960
3014
|
totalTasks: stats.analytics.totalTasks,
|
2961
3015
|
memoryHits: stats.analytics.memoryHits,
|
2962
3016
|
memoryMisses: stats.analytics.memoryMisses,
|
2963
|
-
memoryEffectiveness: Math.round(
|
3017
|
+
memoryEffectiveness: Math.round(
|
3018
|
+
stats.analytics.memoryEffectiveness * 100
|
3019
|
+
),
|
2964
3020
|
averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
|
2965
3021
|
},
|
2966
3022
|
config: stats.config,
|
@@ -3024,7 +3080,9 @@ ${errors}`);
|
|
3024
3080
|
calculateSuccessRate(memory) {
|
3025
3081
|
if (memory.length === 0)
|
3026
3082
|
return 0;
|
3027
|
-
const successCount = memory.filter(
|
3083
|
+
const successCount = memory.filter(
|
3084
|
+
(item) => item.metadata?.success !== false
|
3085
|
+
).length;
|
3028
3086
|
return Math.round(successCount / memory.length * 100);
|
3029
3087
|
}
|
3030
3088
|
calculateAverageExecutionTime(memory) {
|
@@ -3074,13 +3132,236 @@ import {
|
|
3074
3132
|
} from "misoai-shared/fs";
|
3075
3133
|
import { getDebug as getDebug5 } from "misoai-shared/logger";
|
3076
3134
|
import { assert as assert8 } from "misoai-shared/utils";
|
3135
|
+
|
3136
|
+
// src/common/mouse-utils.ts
|
3137
|
+
function generateHumanMousePath(from, to, options = {}) {
|
3138
|
+
const {
|
3139
|
+
steps = 20,
|
3140
|
+
easing = "easeInOut"
|
3141
|
+
} = options;
|
3142
|
+
if (steps <= 1) {
|
3143
|
+
return [to];
|
3144
|
+
}
|
3145
|
+
const path = [];
|
3146
|
+
const controlPoint1 = {
|
3147
|
+
x: from.x + (to.x - from.x) * 0.25 + (Math.random() - 0.5) * 50,
|
3148
|
+
y: from.y + (to.y - from.y) * 0.25 + (Math.random() - 0.5) * 50
|
3149
|
+
};
|
3150
|
+
const controlPoint2 = {
|
3151
|
+
x: from.x + (to.x - from.x) * 0.75 + (Math.random() - 0.5) * 50,
|
3152
|
+
y: from.y + (to.y - from.y) * 0.75 + (Math.random() - 0.5) * 50
|
3153
|
+
};
|
3154
|
+
for (let i = 0; i <= steps; i++) {
|
3155
|
+
const t = i / steps;
|
3156
|
+
const easedT = applyEasing(t, easing);
|
3157
|
+
const point = calculateBezierPoint(from, controlPoint1, controlPoint2, to, easedT);
|
3158
|
+
path.push(point);
|
3159
|
+
}
|
3160
|
+
return path;
|
3161
|
+
}
|
3162
|
+
function calculateBezierPoint(p0, p1, p2, p3, t) {
|
3163
|
+
const oneMinusT = 1 - t;
|
3164
|
+
const oneMinusTSquared = oneMinusT * oneMinusT;
|
3165
|
+
const oneMinusTCubed = oneMinusTSquared * oneMinusT;
|
3166
|
+
const tSquared = t * t;
|
3167
|
+
const tCubed = tSquared * t;
|
3168
|
+
return {
|
3169
|
+
x: oneMinusTCubed * p0.x + 3 * oneMinusTSquared * t * p1.x + 3 * oneMinusT * tSquared * p2.x + tCubed * p3.x,
|
3170
|
+
y: oneMinusTCubed * p0.y + 3 * oneMinusTSquared * t * p1.y + 3 * oneMinusT * tSquared * p2.y + tCubed * p3.y
|
3171
|
+
};
|
3172
|
+
}
|
3173
|
+
function applyEasing(t, easing) {
|
3174
|
+
switch (easing) {
|
3175
|
+
case "linear":
|
3176
|
+
return t;
|
3177
|
+
case "easeIn":
|
3178
|
+
return t * t;
|
3179
|
+
case "easeOut":
|
3180
|
+
return 1 - (1 - t) * (1 - t);
|
3181
|
+
case "easeInOut":
|
3182
|
+
return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
|
3183
|
+
default:
|
3184
|
+
return t;
|
3185
|
+
}
|
3186
|
+
}
|
3187
|
+
function calculateStepDelay(totalDuration, currentStep, totalSteps) {
|
3188
|
+
const baseDelay = totalDuration / totalSteps;
|
3189
|
+
const variation = 0.3;
|
3190
|
+
const randomFactor = 1 + (Math.random() - 0.5) * variation;
|
3191
|
+
return Math.max(1, baseDelay * randomFactor);
|
3192
|
+
}
|
3193
|
+
|
3194
|
+
// src/common/visual-feedback.ts
|
3195
|
+
function getMousePointerScript(x, y, options = {}) {
|
3196
|
+
const {
|
3197
|
+
pointerSize = 20,
|
3198
|
+
pointerColor = "#ff4444",
|
3199
|
+
trailLength = 5,
|
3200
|
+
showTrail = true,
|
3201
|
+
animationDuration = 200
|
3202
|
+
} = options;
|
3203
|
+
return `
|
3204
|
+
(() => {
|
3205
|
+
// Create or update mouse pointer
|
3206
|
+
if (!window.misoaiMousePointer) {
|
3207
|
+
const pointer = document.createElement('div');
|
3208
|
+
pointer.id = 'misoai-mouse-pointer';
|
3209
|
+
pointer.style.cssText = \`
|
3210
|
+
position: fixed;
|
3211
|
+
width: ${pointerSize}px;
|
3212
|
+
height: ${pointerSize}px;
|
3213
|
+
background: ${pointerColor};
|
3214
|
+
border: 2px solid white;
|
3215
|
+
border-radius: 50%;
|
3216
|
+
pointer-events: none;
|
3217
|
+
z-index: 999999;
|
3218
|
+
box-shadow: 0 0 10px rgba(0,0,0,0.3);
|
3219
|
+
transition: all ${animationDuration}ms ease-out;
|
3220
|
+
transform: translate(-50%, -50%);
|
3221
|
+
\`;
|
3222
|
+
document.body.appendChild(pointer);
|
3223
|
+
window.misoaiMousePointer = pointer;
|
3224
|
+
|
3225
|
+
// Initialize trail array
|
3226
|
+
window.misoaiMouseTrail = [];
|
3227
|
+
}
|
3228
|
+
|
3229
|
+
const pointer = window.misoaiMousePointer;
|
3230
|
+
pointer.style.left = ${x}px;
|
3231
|
+
pointer.style.top = ${y}px;
|
3232
|
+
|
3233
|
+
${showTrail ? getTrailScript(x, y, trailLength, pointerColor) : ""}
|
3234
|
+
|
3235
|
+
// Auto-hide after 3 seconds of inactivity
|
3236
|
+
clearTimeout(window.misoaiMouseTimeout);
|
3237
|
+
pointer.style.opacity = '1';
|
3238
|
+
window.misoaiMouseTimeout = setTimeout(() => {
|
3239
|
+
if (pointer) pointer.style.opacity = '0.3';
|
3240
|
+
}, 3000);
|
3241
|
+
})();
|
3242
|
+
`;
|
3243
|
+
}
|
3244
|
+
function getTrailScript(x, y, trailLength, color) {
|
3245
|
+
return `
|
3246
|
+
// Add current position to trail
|
3247
|
+
window.misoaiMouseTrail.push({ x: ${x}, y: ${y}, timestamp: Date.now() });
|
3248
|
+
|
3249
|
+
// Keep only recent trail points
|
3250
|
+
if (window.misoaiMouseTrail.length > ${trailLength}) {
|
3251
|
+
window.misoaiMouseTrail.shift();
|
3252
|
+
}
|
3253
|
+
|
3254
|
+
// Remove old trail elements
|
3255
|
+
document.querySelectorAll('.misoai-mouse-trail').forEach(el => el.remove());
|
3256
|
+
|
3257
|
+
// Create new trail elements
|
3258
|
+
window.misoaiMouseTrail.forEach((point, index) => {
|
3259
|
+
if (index === window.misoaiMouseTrail.length - 1) return; // Skip current position
|
3260
|
+
|
3261
|
+
const trailElement = document.createElement('div');
|
3262
|
+
trailElement.className = 'misoai-mouse-trail';
|
3263
|
+
const opacity = (index + 1) / window.misoaiMouseTrail.length * 0.6;
|
3264
|
+
const size = 8 + (index + 1) / window.misoaiMouseTrail.length * 8;
|
3265
|
+
|
3266
|
+
trailElement.style.cssText = \`
|
3267
|
+
position: fixed;
|
3268
|
+
width: \${size}px;
|
3269
|
+
height: \${size}px;
|
3270
|
+
background: ${color};
|
3271
|
+
border-radius: 50%;
|
3272
|
+
pointer-events: none;
|
3273
|
+
z-index: 999998;
|
3274
|
+
opacity: \${opacity};
|
3275
|
+
transform: translate(-50%, -50%);
|
3276
|
+
left: \${point.x}px;
|
3277
|
+
top: \${point.y}px;
|
3278
|
+
\`;
|
3279
|
+
|
3280
|
+
document.body.appendChild(trailElement);
|
3281
|
+
|
3282
|
+
// Auto-remove trail element after animation
|
3283
|
+
setTimeout(() => {
|
3284
|
+
if (trailElement.parentNode) {
|
3285
|
+
trailElement.parentNode.removeChild(trailElement);
|
3286
|
+
}
|
3287
|
+
}, 1000);
|
3288
|
+
});
|
3289
|
+
`;
|
3290
|
+
}
|
3291
|
+
function getClickAnimationScript(x, y) {
|
3292
|
+
return `
|
3293
|
+
(() => {
|
3294
|
+
const clickRipple = document.createElement('div');
|
3295
|
+
clickRipple.style.cssText = \`
|
3296
|
+
position: fixed;
|
3297
|
+
left: ${x}px;
|
3298
|
+
top: ${y}px;
|
3299
|
+
width: 0;
|
3300
|
+
height: 0;
|
3301
|
+
border: 2px solid #ff4444;
|
3302
|
+
border-radius: 50%;
|
3303
|
+
pointer-events: none;
|
3304
|
+
z-index: 999999;
|
3305
|
+
transform: translate(-50%, -50%);
|
3306
|
+
animation: misoai-click-ripple 0.6s ease-out forwards;
|
3307
|
+
\`;
|
3308
|
+
|
3309
|
+
// Add CSS animation if not exists
|
3310
|
+
if (!document.getElementById('misoai-click-styles')) {
|
3311
|
+
const style = document.createElement('style');
|
3312
|
+
style.id = 'misoai-click-styles';
|
3313
|
+
style.textContent = \`
|
3314
|
+
@keyframes misoai-click-ripple {
|
3315
|
+
0% {
|
3316
|
+
width: 0;
|
3317
|
+
height: 0;
|
3318
|
+
opacity: 1;
|
3319
|
+
}
|
3320
|
+
100% {
|
3321
|
+
width: 40px;
|
3322
|
+
height: 40px;
|
3323
|
+
opacity: 0;
|
3324
|
+
}
|
3325
|
+
}
|
3326
|
+
\`;
|
3327
|
+
document.head.appendChild(style);
|
3328
|
+
}
|
3329
|
+
|
3330
|
+
document.body.appendChild(clickRipple);
|
3331
|
+
|
3332
|
+
// Remove after animation
|
3333
|
+
setTimeout(() => {
|
3334
|
+
if (clickRipple.parentNode) {
|
3335
|
+
clickRipple.parentNode.removeChild(clickRipple);
|
3336
|
+
}
|
3337
|
+
}, 600);
|
3338
|
+
})();
|
3339
|
+
`;
|
3340
|
+
}
|
3341
|
+
|
3342
|
+
// src/puppeteer/base-page.ts
|
3077
3343
|
var debugPage = getDebug5("web:page");
|
3078
3344
|
var Page = class {
|
3079
3345
|
constructor(underlyingPage, pageType, opts) {
|
3346
|
+
this.currentMousePosition = { x: 0, y: 0 };
|
3347
|
+
this.mouseMovementOptions = {
|
3348
|
+
steps: 15,
|
3349
|
+
duration: 300,
|
3350
|
+
easing: "easeInOut",
|
3351
|
+
showVisualFeedback: true
|
3352
|
+
};
|
3353
|
+
this.visualFeedbackOptions = {
|
3354
|
+
pointerSize: 16,
|
3355
|
+
pointerColor: "#ff4444",
|
3356
|
+
trailLength: 4,
|
3357
|
+
showTrail: true,
|
3358
|
+
animationDuration: 150
|
3359
|
+
};
|
3080
3360
|
this.everMoved = false;
|
3081
3361
|
this.underlyingPage = underlyingPage;
|
3082
3362
|
this.pageType = pageType;
|
3083
3363
|
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT2;
|
3364
|
+
this.initializeMousePosition();
|
3084
3365
|
}
|
3085
3366
|
async evaluate(pageFunction, arg) {
|
3086
3367
|
let result;
|
@@ -3099,9 +3380,76 @@ var Page = class {
|
|
3099
3380
|
debugPage("evaluate function end");
|
3100
3381
|
return result;
|
3101
3382
|
}
|
3383
|
+
/**
|
3384
|
+
* Initialize mouse position to center of viewport
|
3385
|
+
*/
|
3386
|
+
async initializeMousePosition() {
|
3387
|
+
try {
|
3388
|
+
const size = await this.size();
|
3389
|
+
this.currentMousePosition = {
|
3390
|
+
x: Math.floor(size.width / 2),
|
3391
|
+
y: Math.floor(size.height / 2)
|
3392
|
+
};
|
3393
|
+
} catch (error) {
|
3394
|
+
this.currentMousePosition = { x: 400, y: 300 };
|
3395
|
+
}
|
3396
|
+
}
|
3102
3397
|
async evaluateJavaScript(script) {
|
3103
3398
|
return this.evaluate(script);
|
3104
3399
|
}
|
3400
|
+
/**
|
3401
|
+
* Show visual feedback for mouse pointer
|
3402
|
+
*/
|
3403
|
+
async showMousePointer(x, y) {
|
3404
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3405
|
+
return;
|
3406
|
+
const script = getMousePointerScript(x, y, this.visualFeedbackOptions);
|
3407
|
+
try {
|
3408
|
+
await this.evaluate(script);
|
3409
|
+
} catch (error) {
|
3410
|
+
debugPage("Visual feedback error:", error);
|
3411
|
+
}
|
3412
|
+
}
|
3413
|
+
/**
|
3414
|
+
* Show click animation
|
3415
|
+
*/
|
3416
|
+
async showClickAnimation(x, y) {
|
3417
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3418
|
+
return;
|
3419
|
+
const script = getClickAnimationScript(x, y);
|
3420
|
+
try {
|
3421
|
+
await this.evaluate(script);
|
3422
|
+
} catch (error) {
|
3423
|
+
debugPage("Click animation error:", error);
|
3424
|
+
}
|
3425
|
+
}
|
3426
|
+
/**
|
3427
|
+
* Move mouse with human-like movement and visual feedback
|
3428
|
+
*/
|
3429
|
+
async moveMouseHumanLike(toX, toY, options) {
|
3430
|
+
const moveOptions = { ...this.mouseMovementOptions, ...options };
|
3431
|
+
const path = generateHumanMousePath(
|
3432
|
+
this.currentMousePosition,
|
3433
|
+
{ x: toX, y: toY },
|
3434
|
+
moveOptions
|
3435
|
+
);
|
3436
|
+
for (let i = 0; i < path.length; i++) {
|
3437
|
+
const point = path[i];
|
3438
|
+
if (moveOptions.showVisualFeedback) {
|
3439
|
+
await this.showMousePointer(point.x, point.y);
|
3440
|
+
}
|
3441
|
+
await this.underlyingPage.mouse.move(point.x, point.y);
|
3442
|
+
this.currentMousePosition = point;
|
3443
|
+
if (i < path.length - 1) {
|
3444
|
+
const delay = calculateStepDelay(
|
3445
|
+
moveOptions.duration || 300,
|
3446
|
+
i,
|
3447
|
+
path.length
|
3448
|
+
);
|
3449
|
+
await sleep2(delay);
|
3450
|
+
}
|
3451
|
+
}
|
3452
|
+
}
|
3105
3453
|
async waitForNavigation() {
|
3106
3454
|
if (this.pageType === "puppeteer" || this.pageType === "playwright") {
|
3107
3455
|
debugPage("waitForNavigation begin");
|
@@ -3191,7 +3539,8 @@ var Page = class {
|
|
3191
3539
|
return {
|
3192
3540
|
click: async (x, y, options) => {
|
3193
3541
|
await this.mouse.move(x, y);
|
3194
|
-
this.
|
3542
|
+
await this.showClickAnimation(x, y);
|
3543
|
+
await this.underlyingPage.mouse.click(x, y, {
|
3195
3544
|
button: options?.button || "left",
|
3196
3545
|
count: options?.count || 1
|
3197
3546
|
});
|
@@ -3211,27 +3560,23 @@ var Page = class {
|
|
3211
3560
|
},
|
3212
3561
|
move: async (x, y) => {
|
3213
3562
|
this.everMoved = true;
|
3214
|
-
|
3563
|
+
await this.moveMouseHumanLike(x, y);
|
3564
|
+
return Promise.resolve();
|
3215
3565
|
},
|
3216
3566
|
drag: async (from, to) => {
|
3217
3567
|
if (this.pageType === "puppeteer") {
|
3218
|
-
await this.
|
3219
|
-
|
3220
|
-
|
3221
|
-
|
3222
|
-
|
3223
|
-
|
3224
|
-
|
3225
|
-
|
3226
|
-
}
|
3227
|
-
);
|
3568
|
+
await this.mouse.move(from.x, from.y);
|
3569
|
+
await this.underlyingPage.mouse.down();
|
3570
|
+
await this.moveMouseHumanLike(to.x, to.y, {
|
3571
|
+
duration: 500,
|
3572
|
+
// Slower for drag operations
|
3573
|
+
steps: 25
|
3574
|
+
});
|
3575
|
+
await this.underlyingPage.mouse.up();
|
3228
3576
|
} else if (this.pageType === "playwright") {
|
3229
|
-
await this.
|
3230
|
-
from.x,
|
3231
|
-
from.y
|
3232
|
-
);
|
3577
|
+
await this.mouse.move(from.x, from.y);
|
3233
3578
|
await this.underlyingPage.mouse.down();
|
3234
|
-
await this.
|
3579
|
+
await this.mouse.move(to.x, to.y);
|
3235
3580
|
await this.underlyingPage.mouse.up();
|
3236
3581
|
}
|
3237
3582
|
}
|
@@ -3294,6 +3639,18 @@ var Page = class {
|
|
3294
3639
|
await this.mouse.move(targetX, targetY);
|
3295
3640
|
}
|
3296
3641
|
}
|
3642
|
+
/**
|
3643
|
+
* Configure mouse movement behavior
|
3644
|
+
*/
|
3645
|
+
configureMouseMovement(options) {
|
3646
|
+
this.mouseMovementOptions = { ...this.mouseMovementOptions, ...options };
|
3647
|
+
}
|
3648
|
+
/**
|
3649
|
+
* Configure visual feedback
|
3650
|
+
*/
|
3651
|
+
configureVisualFeedback(options) {
|
3652
|
+
this.visualFeedbackOptions = { ...this.visualFeedbackOptions, ...options };
|
3653
|
+
}
|
3297
3654
|
async scrollUntilTop(startingPoint) {
|
3298
3655
|
await this.moveToPointBeforeScroll(startingPoint);
|
3299
3656
|
return this.mouse.wheel(0, -9999999);
|