misoai-web 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent.js +108 -50
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +110 -52
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +109 -51
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +425 -68
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +111 -53
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js +4 -4
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +108 -50
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +425 -68
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +424 -67
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +424 -67
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/utils.js +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +108 -50
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +110 -52
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +109 -51
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +425 -68
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +111 -53
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js +4 -4
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +108 -50
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +425 -68
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +424 -67
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +424 -67
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +1 -1
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-9b472ffb.d.ts → browser-f205f69d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +2 -2
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-ed0ecb44.d.ts → page-c5452809.d.ts} +45 -0
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +2 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +2 -2
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +18 -54
@@ -311,8 +311,8 @@ var ScriptPlayer = class {
|
|
311
311
|
import yaml from "js-yaml";
|
312
312
|
|
313
313
|
// src/yaml/utils.ts
|
314
|
-
import { assert as assert2 } from "misoai-shared/utils";
|
315
314
|
import yaml2 from "js-yaml";
|
315
|
+
import { assert as assert2 } from "misoai-shared/utils";
|
316
316
|
function interpolateEnvVars(content) {
|
317
317
|
return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
|
318
318
|
const value = process.env[envVar.trim()];
|
@@ -473,6 +473,7 @@ function paramStr(task) {
|
|
473
473
|
}
|
474
474
|
|
475
475
|
// src/common/utils.ts
|
476
|
+
import dayjs from "dayjs";
|
476
477
|
import { elementByPositionWithElementInfo } from "misoai-core/ai-model";
|
477
478
|
import { uploadTestInfoToServer } from "misoai-core/utils";
|
478
479
|
import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "misoai-shared/env";
|
@@ -484,7 +485,6 @@ import {
|
|
484
485
|
} from "misoai-shared/extractor";
|
485
486
|
import { resizeImgBase64 } from "misoai-shared/img";
|
486
487
|
import { assert as assert3, logMsg, uuid } from "misoai-shared/utils";
|
487
|
-
import dayjs from "dayjs";
|
488
488
|
|
489
489
|
// src/web-element.ts
|
490
490
|
var WebElementInfo = class {
|
@@ -653,8 +653,12 @@ var WorkflowMemory = class {
|
|
653
653
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
654
654
|
workflow.memory = [...memory];
|
655
655
|
workflow.metadata.totalSteps = workflow.steps.length;
|
656
|
-
workflow.metadata.completedSteps = workflow.steps.filter(
|
657
|
-
|
656
|
+
workflow.metadata.completedSteps = workflow.steps.filter(
|
657
|
+
(s) => s.status === "completed"
|
658
|
+
).length;
|
659
|
+
workflow.metadata.failedSteps = workflow.steps.filter(
|
660
|
+
(s) => s.status === "failed"
|
661
|
+
).length;
|
658
662
|
this.workflows.set(workflowId, workflow);
|
659
663
|
this.enforceRetentionPolicy();
|
660
664
|
}
|
@@ -665,7 +669,9 @@ var WorkflowMemory = class {
|
|
665
669
|
const workflow = this.workflows.get(workflowId) || this.createEmptyWorkflowData(workflowId);
|
666
670
|
workflow.context = { ...workflow.context, ...context };
|
667
671
|
if (context.currentStep) {
|
668
|
-
const existingStep = workflow.steps.find(
|
672
|
+
const existingStep = workflow.steps.find(
|
673
|
+
(s) => s.stepName === context.currentStep
|
674
|
+
);
|
669
675
|
if (!existingStep) {
|
670
676
|
workflow.steps.push({
|
671
677
|
stepId: `step_${workflow.steps.length + 1}`,
|
@@ -710,7 +716,9 @@ var WorkflowMemory = class {
|
|
710
716
|
enforceRetentionPolicy() {
|
711
717
|
const maxWorkflows = 10;
|
712
718
|
if (this.workflows.size > maxWorkflows) {
|
713
|
-
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
719
|
+
const sortedWorkflows = Array.from(this.workflows.entries()).sort(
|
720
|
+
([, a], [, b]) => (b.metadata.endTime || b.metadata.startTime) - (a.metadata.endTime || a.metadata.startTime)
|
721
|
+
);
|
714
722
|
const toDelete = sortedWorkflows.slice(maxWorkflows);
|
715
723
|
toDelete.forEach(([workflowId]) => this.workflows.delete(workflowId));
|
716
724
|
}
|
@@ -1277,6 +1285,7 @@ var PageTaskExecutor = class {
|
|
1277
1285
|
actions: [],
|
1278
1286
|
more_actions_needed_by_instruction: false,
|
1279
1287
|
log: "",
|
1288
|
+
summary: "Loaded YAML workflow configuration",
|
1280
1289
|
yamlString
|
1281
1290
|
},
|
1282
1291
|
cache: {
|
@@ -1378,6 +1387,7 @@ var PageTaskExecutor = class {
|
|
1378
1387
|
actions: finalActions,
|
1379
1388
|
more_actions_needed_by_instruction,
|
1380
1389
|
log: log2,
|
1390
|
+
summary: planResult.summary || "Generated action plan from user instruction",
|
1381
1391
|
yamlFlow: planResult.yamlFlow
|
1382
1392
|
},
|
1383
1393
|
cache: {
|
@@ -1433,6 +1443,7 @@ var PageTaskExecutor = class {
|
|
1433
1443
|
actionType: actions[0].type,
|
1434
1444
|
more_actions_needed_by_instruction: true,
|
1435
1445
|
log: "",
|
1446
|
+
summary: action_summary || "Generated VLM action plan",
|
1436
1447
|
yamlFlow: planResult.yamlFlow
|
1437
1448
|
},
|
1438
1449
|
cache: {
|
@@ -1449,7 +1460,9 @@ var PageTaskExecutor = class {
|
|
1449
1460
|
*/
|
1450
1461
|
getPersistentExecutor() {
|
1451
1462
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1452
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1463
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1464
|
+
this.sessionContext.workflowId
|
1465
|
+
);
|
1453
1466
|
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1454
1467
|
onTaskStart: this.onTaskStartCallback,
|
1455
1468
|
initialMemory: previousMemory
|
@@ -1478,7 +1491,9 @@ var PageTaskExecutor = class {
|
|
1478
1491
|
if (this.persistentExecutor) {
|
1479
1492
|
this.persistentExecutor.clearMemory();
|
1480
1493
|
}
|
1481
|
-
this.workflowMemory.clearWorkflow(
|
1494
|
+
this.workflowMemory.clearWorkflow(
|
1495
|
+
this.sessionContext.workflowId || "default"
|
1496
|
+
);
|
1482
1497
|
}
|
1483
1498
|
/**
|
1484
1499
|
* Mevcut hafızayı döndürür
|
@@ -1490,7 +1505,9 @@ var PageTaskExecutor = class {
|
|
1490
1505
|
* İş akışı hafızasını döndürür
|
1491
1506
|
*/
|
1492
1507
|
getWorkflowMemory() {
|
1493
|
-
return this.workflowMemory.getWorkflowData(
|
1508
|
+
return this.workflowMemory.getWorkflowData(
|
1509
|
+
this.sessionContext.workflowId || "default"
|
1510
|
+
);
|
1494
1511
|
}
|
1495
1512
|
/**
|
1496
1513
|
* Hafıza istatistiklerini döndürür
|
@@ -1498,7 +1515,13 @@ var PageTaskExecutor = class {
|
|
1498
1515
|
getMemoryStats() {
|
1499
1516
|
return this.persistentExecutor?.getMemoryStats() || {
|
1500
1517
|
totalItems: 0,
|
1501
|
-
analytics: {
|
1518
|
+
analytics: {
|
1519
|
+
totalTasks: 0,
|
1520
|
+
memoryHits: 0,
|
1521
|
+
memoryMisses: 0,
|
1522
|
+
averageMemorySize: 0,
|
1523
|
+
memoryEffectiveness: 0
|
1524
|
+
},
|
1502
1525
|
config: this.memoryConfig
|
1503
1526
|
};
|
1504
1527
|
}
|
@@ -1534,11 +1557,14 @@ var PageTaskExecutor = class {
|
|
1534
1557
|
let taskExecutor;
|
1535
1558
|
if (useMemory) {
|
1536
1559
|
taskExecutor = this.getPersistentExecutor();
|
1537
|
-
this.workflowMemory.updateWorkflowContext(
|
1538
|
-
|
1539
|
-
|
1540
|
-
|
1541
|
-
|
1560
|
+
this.workflowMemory.updateWorkflowContext(
|
1561
|
+
{
|
1562
|
+
currentStep: title,
|
1563
|
+
pageInfo: this.sessionContext.pageInfo,
|
1564
|
+
timestamp: Date.now()
|
1565
|
+
},
|
1566
|
+
this.sessionContext.workflowId || "default"
|
1567
|
+
);
|
1542
1568
|
} else {
|
1543
1569
|
taskExecutor = new Executor(title, {
|
1544
1570
|
onTaskStart: this.onTaskStartCallback
|
@@ -1904,14 +1930,19 @@ var PageTaskExecutor = class {
|
|
1904
1930
|
*/
|
1905
1931
|
addToMemory(memoryItem) {
|
1906
1932
|
if (!this.persistentExecutor || this.persistentExecutor.status === "error") {
|
1907
|
-
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1933
|
+
const previousMemory = this.workflowMemory.getWorkflowMemory(
|
1934
|
+
this.sessionContext.workflowId
|
1935
|
+
);
|
1908
1936
|
this.persistentExecutor = new Executor("Persistent Task Executor", {
|
1909
1937
|
onTaskStart: this.onTaskStartCallback,
|
1910
1938
|
initialMemory: previousMemory
|
1911
1939
|
});
|
1912
1940
|
}
|
1913
1941
|
this.persistentExecutor.memoryStore?.add(memoryItem);
|
1914
|
-
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1942
|
+
this.persistentExecutor.memoryAnalytics?.recordMemoryOperation(
|
1943
|
+
"add",
|
1944
|
+
memoryItem
|
1945
|
+
);
|
1915
1946
|
}
|
1916
1947
|
};
|
1917
1948
|
|
@@ -2000,14 +2031,14 @@ function buildPlans(type, locateParam, param) {
|
|
2000
2031
|
import assert6 from "assert";
|
2001
2032
|
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
2002
2033
|
import { dirname as dirname2, join as join2 } from "path";
|
2034
|
+
import yaml3 from "js-yaml";
|
2003
2035
|
import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
|
2004
2036
|
import { getDebug as getDebug3 } from "misoai-shared/logger";
|
2005
2037
|
import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
|
2006
|
-
import yaml3 from "js-yaml";
|
2007
2038
|
import semver from "semver";
|
2008
2039
|
|
2009
2040
|
// package.json
|
2010
|
-
var version = "1.6.
|
2041
|
+
var version = "1.6.2";
|
2011
2042
|
|
2012
2043
|
// src/common/task-cache.ts
|
2013
2044
|
var debug3 = getDebug3("cache");
|
@@ -2302,22 +2333,28 @@ var PageAgent = class {
|
|
2302
2333
|
const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
|
2303
2334
|
const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
|
2304
2335
|
const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
|
2305
|
-
const planningTasks = executor.tasks.filter(
|
2306
|
-
|
2336
|
+
const planningTasks = executor.tasks.filter(
|
2337
|
+
(task) => task.type === "Planning"
|
2338
|
+
);
|
2339
|
+
const insightTasks = executor.tasks.filter(
|
2340
|
+
(task) => task.type === "Insight"
|
2341
|
+
);
|
2307
2342
|
const actionTasks = executor.tasks.filter((task) => task.type === "Action");
|
2308
2343
|
const planning = planningTasks.length > 0 ? {
|
2309
2344
|
type: "Planning",
|
2310
|
-
description:
|
2345
|
+
description: "Planning for task execution",
|
2311
2346
|
steps: planningTasks.map((task) => task.thought || "Planning step")
|
2312
2347
|
} : void 0;
|
2313
2348
|
const insight = insightTasks.length > 0 ? {
|
2314
2349
|
type: "Insight",
|
2315
|
-
description:
|
2316
|
-
elements: insightTasks.map(
|
2350
|
+
description: "Insight for task execution",
|
2351
|
+
elements: insightTasks.map(
|
2352
|
+
(task) => task.thought || "Insight element"
|
2353
|
+
)
|
2317
2354
|
} : void 0;
|
2318
2355
|
const action = actionTasks.length > 0 ? {
|
2319
2356
|
type: "Action",
|
2320
|
-
description:
|
2357
|
+
description: "Action for task execution",
|
2321
2358
|
result: lastTask?.output
|
2322
2359
|
} : void 0;
|
2323
2360
|
const actionDetails = executor.tasks.map((task) => ({
|
@@ -2651,7 +2688,10 @@ ${memoryContext}` : void 0;
|
|
2651
2688
|
}
|
2652
2689
|
const memoryContext = this.getMemoryAsContext();
|
2653
2690
|
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2654
|
-
const { output, executor } = await this.taskExecutor.assert(
|
2691
|
+
const { output, executor } = await this.taskExecutor.assert(
|
2692
|
+
assertionWithContext,
|
2693
|
+
memoryContext
|
2694
|
+
);
|
2655
2695
|
const metadata = this.afterTaskRunning(executor, true);
|
2656
2696
|
if (output && opt?.keepRawResponse) {
|
2657
2697
|
return {
|
@@ -2690,7 +2730,10 @@ A complex CAPTCHA typically has one or more of these characteristics:
|
|
2690
2730
|
Return only "complex" or "simple" based on your analysis.
|
2691
2731
|
`;
|
2692
2732
|
const complexityMsgs = [
|
2693
|
-
{
|
2733
|
+
{
|
2734
|
+
role: "system",
|
2735
|
+
content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity."
|
2736
|
+
},
|
2694
2737
|
{
|
2695
2738
|
role: "user",
|
2696
2739
|
content: [
|
@@ -2714,7 +2757,12 @@ Return only "complex" or "simple" based on your analysis.
|
|
2714
2757
|
);
|
2715
2758
|
const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
|
2716
2759
|
shouldUseDeepThink = responseText.includes("complex");
|
2717
|
-
debug4(
|
2760
|
+
debug4(
|
2761
|
+
"CAPTCHA complexity analysis:",
|
2762
|
+
responseText,
|
2763
|
+
"Using deep think:",
|
2764
|
+
shouldUseDeepThink
|
2765
|
+
);
|
2718
2766
|
} catch (error) {
|
2719
2767
|
debug4("Failed to analyze CAPTCHA complexity:", error);
|
2720
2768
|
}
|
@@ -2731,7 +2779,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2731
2779
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2732
2780
|
} else if (action.type === "input" && action.value) {
|
2733
2781
|
if (action.target) {
|
2734
|
-
await this.aiInput(action.value, action.target, {
|
2782
|
+
await this.aiInput(action.value, action.target, {
|
2783
|
+
deepThink: shouldUseDeepThink
|
2784
|
+
});
|
2735
2785
|
}
|
2736
2786
|
} else if (action.type === "verify" && action.target) {
|
2737
2787
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
@@ -2743,7 +2793,9 @@ Return only "complex" or "simple" based on your analysis.
|
|
2743
2793
|
if (action.coordinates) {
|
2744
2794
|
const x = action.coordinates[0];
|
2745
2795
|
const y = action.coordinates[1];
|
2746
|
-
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2796
|
+
await this.aiTap(`element at coordinates (${x}, ${y})`, {
|
2797
|
+
deepThink: shouldUseDeepThink
|
2798
|
+
});
|
2747
2799
|
} else if (action.target) {
|
2748
2800
|
await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
|
2749
2801
|
}
|
@@ -2895,25 +2947,27 @@ ${errors}`);
|
|
2895
2947
|
const executionDump = {
|
2896
2948
|
name: screenshotTitle,
|
2897
2949
|
description: content,
|
2898
|
-
tasks: [
|
2899
|
-
|
2900
|
-
|
2901
|
-
|
2902
|
-
|
2903
|
-
|
2904
|
-
|
2905
|
-
|
2906
|
-
|
2907
|
-
|
2908
|
-
|
2909
|
-
|
2910
|
-
|
2911
|
-
|
2912
|
-
|
2913
|
-
|
2914
|
-
|
2950
|
+
tasks: [
|
2951
|
+
{
|
2952
|
+
type: "Screenshot",
|
2953
|
+
subType: "log",
|
2954
|
+
status: "finished",
|
2955
|
+
executor: null,
|
2956
|
+
param: {
|
2957
|
+
title: screenshotTitle,
|
2958
|
+
content
|
2959
|
+
},
|
2960
|
+
output: {
|
2961
|
+
screenshot
|
2962
|
+
},
|
2963
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2964
|
+
timing: {
|
2965
|
+
start: Date.now(),
|
2966
|
+
end: Date.now(),
|
2967
|
+
cost: 0
|
2968
|
+
}
|
2915
2969
|
}
|
2916
|
-
|
2970
|
+
],
|
2917
2971
|
sdkVersion: "1.0.0",
|
2918
2972
|
logTime: Date.now(),
|
2919
2973
|
model_name: "screenshot"
|
@@ -2965,7 +3019,9 @@ ${errors}`);
|
|
2965
3019
|
totalTasks: stats.analytics.totalTasks,
|
2966
3020
|
memoryHits: stats.analytics.memoryHits,
|
2967
3021
|
memoryMisses: stats.analytics.memoryMisses,
|
2968
|
-
memoryEffectiveness: Math.round(
|
3022
|
+
memoryEffectiveness: Math.round(
|
3023
|
+
stats.analytics.memoryEffectiveness * 100
|
3024
|
+
),
|
2969
3025
|
averageMemorySize: Math.round(stats.analytics.averageMemorySize * 100) / 100
|
2970
3026
|
},
|
2971
3027
|
config: stats.config,
|
@@ -3029,7 +3085,9 @@ ${errors}`);
|
|
3029
3085
|
calculateSuccessRate(memory) {
|
3030
3086
|
if (memory.length === 0)
|
3031
3087
|
return 0;
|
3032
|
-
const successCount = memory.filter(
|
3088
|
+
const successCount = memory.filter(
|
3089
|
+
(item) => item.metadata?.success !== false
|
3090
|
+
).length;
|
3033
3091
|
return Math.round(successCount / memory.length * 100);
|
3034
3092
|
}
|
3035
3093
|
calculateAverageExecutionTime(memory) {
|
@@ -3079,13 +3137,236 @@ import {
|
|
3079
3137
|
} from "misoai-shared/fs";
|
3080
3138
|
import { getDebug as getDebug5 } from "misoai-shared/logger";
|
3081
3139
|
import { assert as assert8 } from "misoai-shared/utils";
|
3140
|
+
|
3141
|
+
// src/common/mouse-utils.ts
|
3142
|
+
function generateHumanMousePath(from, to, options = {}) {
|
3143
|
+
const {
|
3144
|
+
steps = 20,
|
3145
|
+
easing = "easeInOut"
|
3146
|
+
} = options;
|
3147
|
+
if (steps <= 1) {
|
3148
|
+
return [to];
|
3149
|
+
}
|
3150
|
+
const path = [];
|
3151
|
+
const controlPoint1 = {
|
3152
|
+
x: from.x + (to.x - from.x) * 0.25 + (Math.random() - 0.5) * 50,
|
3153
|
+
y: from.y + (to.y - from.y) * 0.25 + (Math.random() - 0.5) * 50
|
3154
|
+
};
|
3155
|
+
const controlPoint2 = {
|
3156
|
+
x: from.x + (to.x - from.x) * 0.75 + (Math.random() - 0.5) * 50,
|
3157
|
+
y: from.y + (to.y - from.y) * 0.75 + (Math.random() - 0.5) * 50
|
3158
|
+
};
|
3159
|
+
for (let i = 0; i <= steps; i++) {
|
3160
|
+
const t = i / steps;
|
3161
|
+
const easedT = applyEasing(t, easing);
|
3162
|
+
const point = calculateBezierPoint(from, controlPoint1, controlPoint2, to, easedT);
|
3163
|
+
path.push(point);
|
3164
|
+
}
|
3165
|
+
return path;
|
3166
|
+
}
|
3167
|
+
function calculateBezierPoint(p0, p1, p2, p3, t) {
|
3168
|
+
const oneMinusT = 1 - t;
|
3169
|
+
const oneMinusTSquared = oneMinusT * oneMinusT;
|
3170
|
+
const oneMinusTCubed = oneMinusTSquared * oneMinusT;
|
3171
|
+
const tSquared = t * t;
|
3172
|
+
const tCubed = tSquared * t;
|
3173
|
+
return {
|
3174
|
+
x: oneMinusTCubed * p0.x + 3 * oneMinusTSquared * t * p1.x + 3 * oneMinusT * tSquared * p2.x + tCubed * p3.x,
|
3175
|
+
y: oneMinusTCubed * p0.y + 3 * oneMinusTSquared * t * p1.y + 3 * oneMinusT * tSquared * p2.y + tCubed * p3.y
|
3176
|
+
};
|
3177
|
+
}
|
3178
|
+
function applyEasing(t, easing) {
|
3179
|
+
switch (easing) {
|
3180
|
+
case "linear":
|
3181
|
+
return t;
|
3182
|
+
case "easeIn":
|
3183
|
+
return t * t;
|
3184
|
+
case "easeOut":
|
3185
|
+
return 1 - (1 - t) * (1 - t);
|
3186
|
+
case "easeInOut":
|
3187
|
+
return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
|
3188
|
+
default:
|
3189
|
+
return t;
|
3190
|
+
}
|
3191
|
+
}
|
3192
|
+
function calculateStepDelay(totalDuration, currentStep, totalSteps) {
|
3193
|
+
const baseDelay = totalDuration / totalSteps;
|
3194
|
+
const variation = 0.3;
|
3195
|
+
const randomFactor = 1 + (Math.random() - 0.5) * variation;
|
3196
|
+
return Math.max(1, baseDelay * randomFactor);
|
3197
|
+
}
|
3198
|
+
|
3199
|
+
// src/common/visual-feedback.ts
|
3200
|
+
function getMousePointerScript(x, y, options = {}) {
|
3201
|
+
const {
|
3202
|
+
pointerSize = 20,
|
3203
|
+
pointerColor = "#ff4444",
|
3204
|
+
trailLength = 5,
|
3205
|
+
showTrail = true,
|
3206
|
+
animationDuration = 200
|
3207
|
+
} = options;
|
3208
|
+
return `
|
3209
|
+
(() => {
|
3210
|
+
// Create or update mouse pointer
|
3211
|
+
if (!window.misoaiMousePointer) {
|
3212
|
+
const pointer = document.createElement('div');
|
3213
|
+
pointer.id = 'misoai-mouse-pointer';
|
3214
|
+
pointer.style.cssText = \`
|
3215
|
+
position: fixed;
|
3216
|
+
width: ${pointerSize}px;
|
3217
|
+
height: ${pointerSize}px;
|
3218
|
+
background: ${pointerColor};
|
3219
|
+
border: 2px solid white;
|
3220
|
+
border-radius: 50%;
|
3221
|
+
pointer-events: none;
|
3222
|
+
z-index: 999999;
|
3223
|
+
box-shadow: 0 0 10px rgba(0,0,0,0.3);
|
3224
|
+
transition: all ${animationDuration}ms ease-out;
|
3225
|
+
transform: translate(-50%, -50%);
|
3226
|
+
\`;
|
3227
|
+
document.body.appendChild(pointer);
|
3228
|
+
window.misoaiMousePointer = pointer;
|
3229
|
+
|
3230
|
+
// Initialize trail array
|
3231
|
+
window.misoaiMouseTrail = [];
|
3232
|
+
}
|
3233
|
+
|
3234
|
+
const pointer = window.misoaiMousePointer;
|
3235
|
+
pointer.style.left = ${x}px;
|
3236
|
+
pointer.style.top = ${y}px;
|
3237
|
+
|
3238
|
+
${showTrail ? getTrailScript(x, y, trailLength, pointerColor) : ""}
|
3239
|
+
|
3240
|
+
// Auto-hide after 3 seconds of inactivity
|
3241
|
+
clearTimeout(window.misoaiMouseTimeout);
|
3242
|
+
pointer.style.opacity = '1';
|
3243
|
+
window.misoaiMouseTimeout = setTimeout(() => {
|
3244
|
+
if (pointer) pointer.style.opacity = '0.3';
|
3245
|
+
}, 3000);
|
3246
|
+
})();
|
3247
|
+
`;
|
3248
|
+
}
|
3249
|
+
function getTrailScript(x, y, trailLength, color) {
|
3250
|
+
return `
|
3251
|
+
// Add current position to trail
|
3252
|
+
window.misoaiMouseTrail.push({ x: ${x}, y: ${y}, timestamp: Date.now() });
|
3253
|
+
|
3254
|
+
// Keep only recent trail points
|
3255
|
+
if (window.misoaiMouseTrail.length > ${trailLength}) {
|
3256
|
+
window.misoaiMouseTrail.shift();
|
3257
|
+
}
|
3258
|
+
|
3259
|
+
// Remove old trail elements
|
3260
|
+
document.querySelectorAll('.misoai-mouse-trail').forEach(el => el.remove());
|
3261
|
+
|
3262
|
+
// Create new trail elements
|
3263
|
+
window.misoaiMouseTrail.forEach((point, index) => {
|
3264
|
+
if (index === window.misoaiMouseTrail.length - 1) return; // Skip current position
|
3265
|
+
|
3266
|
+
const trailElement = document.createElement('div');
|
3267
|
+
trailElement.className = 'misoai-mouse-trail';
|
3268
|
+
const opacity = (index + 1) / window.misoaiMouseTrail.length * 0.6;
|
3269
|
+
const size = 8 + (index + 1) / window.misoaiMouseTrail.length * 8;
|
3270
|
+
|
3271
|
+
trailElement.style.cssText = \`
|
3272
|
+
position: fixed;
|
3273
|
+
width: \${size}px;
|
3274
|
+
height: \${size}px;
|
3275
|
+
background: ${color};
|
3276
|
+
border-radius: 50%;
|
3277
|
+
pointer-events: none;
|
3278
|
+
z-index: 999998;
|
3279
|
+
opacity: \${opacity};
|
3280
|
+
transform: translate(-50%, -50%);
|
3281
|
+
left: \${point.x}px;
|
3282
|
+
top: \${point.y}px;
|
3283
|
+
\`;
|
3284
|
+
|
3285
|
+
document.body.appendChild(trailElement);
|
3286
|
+
|
3287
|
+
// Auto-remove trail element after animation
|
3288
|
+
setTimeout(() => {
|
3289
|
+
if (trailElement.parentNode) {
|
3290
|
+
trailElement.parentNode.removeChild(trailElement);
|
3291
|
+
}
|
3292
|
+
}, 1000);
|
3293
|
+
});
|
3294
|
+
`;
|
3295
|
+
}
|
3296
|
+
function getClickAnimationScript(x, y) {
|
3297
|
+
return `
|
3298
|
+
(() => {
|
3299
|
+
const clickRipple = document.createElement('div');
|
3300
|
+
clickRipple.style.cssText = \`
|
3301
|
+
position: fixed;
|
3302
|
+
left: ${x}px;
|
3303
|
+
top: ${y}px;
|
3304
|
+
width: 0;
|
3305
|
+
height: 0;
|
3306
|
+
border: 2px solid #ff4444;
|
3307
|
+
border-radius: 50%;
|
3308
|
+
pointer-events: none;
|
3309
|
+
z-index: 999999;
|
3310
|
+
transform: translate(-50%, -50%);
|
3311
|
+
animation: misoai-click-ripple 0.6s ease-out forwards;
|
3312
|
+
\`;
|
3313
|
+
|
3314
|
+
// Add CSS animation if not exists
|
3315
|
+
if (!document.getElementById('misoai-click-styles')) {
|
3316
|
+
const style = document.createElement('style');
|
3317
|
+
style.id = 'misoai-click-styles';
|
3318
|
+
style.textContent = \`
|
3319
|
+
@keyframes misoai-click-ripple {
|
3320
|
+
0% {
|
3321
|
+
width: 0;
|
3322
|
+
height: 0;
|
3323
|
+
opacity: 1;
|
3324
|
+
}
|
3325
|
+
100% {
|
3326
|
+
width: 40px;
|
3327
|
+
height: 40px;
|
3328
|
+
opacity: 0;
|
3329
|
+
}
|
3330
|
+
}
|
3331
|
+
\`;
|
3332
|
+
document.head.appendChild(style);
|
3333
|
+
}
|
3334
|
+
|
3335
|
+
document.body.appendChild(clickRipple);
|
3336
|
+
|
3337
|
+
// Remove after animation
|
3338
|
+
setTimeout(() => {
|
3339
|
+
if (clickRipple.parentNode) {
|
3340
|
+
clickRipple.parentNode.removeChild(clickRipple);
|
3341
|
+
}
|
3342
|
+
}, 600);
|
3343
|
+
})();
|
3344
|
+
`;
|
3345
|
+
}
|
3346
|
+
|
3347
|
+
// src/puppeteer/base-page.ts
|
3082
3348
|
var debugPage = getDebug5("web:page");
|
3083
3349
|
var Page = class {
|
3084
3350
|
constructor(underlyingPage, pageType, opts) {
|
3351
|
+
this.currentMousePosition = { x: 0, y: 0 };
|
3352
|
+
this.mouseMovementOptions = {
|
3353
|
+
steps: 15,
|
3354
|
+
duration: 300,
|
3355
|
+
easing: "easeInOut",
|
3356
|
+
showVisualFeedback: true
|
3357
|
+
};
|
3358
|
+
this.visualFeedbackOptions = {
|
3359
|
+
pointerSize: 16,
|
3360
|
+
pointerColor: "#ff4444",
|
3361
|
+
trailLength: 4,
|
3362
|
+
showTrail: true,
|
3363
|
+
animationDuration: 150
|
3364
|
+
};
|
3085
3365
|
this.everMoved = false;
|
3086
3366
|
this.underlyingPage = underlyingPage;
|
3087
3367
|
this.pageType = pageType;
|
3088
3368
|
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT2;
|
3369
|
+
this.initializeMousePosition();
|
3089
3370
|
}
|
3090
3371
|
async evaluate(pageFunction, arg) {
|
3091
3372
|
let result;
|
@@ -3104,9 +3385,76 @@ var Page = class {
|
|
3104
3385
|
debugPage("evaluate function end");
|
3105
3386
|
return result;
|
3106
3387
|
}
|
3388
|
+
/**
|
3389
|
+
* Initialize mouse position to center of viewport
|
3390
|
+
*/
|
3391
|
+
async initializeMousePosition() {
|
3392
|
+
try {
|
3393
|
+
const size = await this.size();
|
3394
|
+
this.currentMousePosition = {
|
3395
|
+
x: Math.floor(size.width / 2),
|
3396
|
+
y: Math.floor(size.height / 2)
|
3397
|
+
};
|
3398
|
+
} catch (error) {
|
3399
|
+
this.currentMousePosition = { x: 400, y: 300 };
|
3400
|
+
}
|
3401
|
+
}
|
3107
3402
|
async evaluateJavaScript(script) {
|
3108
3403
|
return this.evaluate(script);
|
3109
3404
|
}
|
3405
|
+
/**
|
3406
|
+
* Show visual feedback for mouse pointer
|
3407
|
+
*/
|
3408
|
+
async showMousePointer(x, y) {
|
3409
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3410
|
+
return;
|
3411
|
+
const script = getMousePointerScript(x, y, this.visualFeedbackOptions);
|
3412
|
+
try {
|
3413
|
+
await this.evaluate(script);
|
3414
|
+
} catch (error) {
|
3415
|
+
debugPage("Visual feedback error:", error);
|
3416
|
+
}
|
3417
|
+
}
|
3418
|
+
/**
|
3419
|
+
* Show click animation
|
3420
|
+
*/
|
3421
|
+
async showClickAnimation(x, y) {
|
3422
|
+
if (!this.mouseMovementOptions.showVisualFeedback)
|
3423
|
+
return;
|
3424
|
+
const script = getClickAnimationScript(x, y);
|
3425
|
+
try {
|
3426
|
+
await this.evaluate(script);
|
3427
|
+
} catch (error) {
|
3428
|
+
debugPage("Click animation error:", error);
|
3429
|
+
}
|
3430
|
+
}
|
3431
|
+
/**
|
3432
|
+
* Move mouse with human-like movement and visual feedback
|
3433
|
+
*/
|
3434
|
+
async moveMouseHumanLike(toX, toY, options) {
|
3435
|
+
const moveOptions = { ...this.mouseMovementOptions, ...options };
|
3436
|
+
const path = generateHumanMousePath(
|
3437
|
+
this.currentMousePosition,
|
3438
|
+
{ x: toX, y: toY },
|
3439
|
+
moveOptions
|
3440
|
+
);
|
3441
|
+
for (let i = 0; i < path.length; i++) {
|
3442
|
+
const point = path[i];
|
3443
|
+
if (moveOptions.showVisualFeedback) {
|
3444
|
+
await this.showMousePointer(point.x, point.y);
|
3445
|
+
}
|
3446
|
+
await this.underlyingPage.mouse.move(point.x, point.y);
|
3447
|
+
this.currentMousePosition = point;
|
3448
|
+
if (i < path.length - 1) {
|
3449
|
+
const delay = calculateStepDelay(
|
3450
|
+
moveOptions.duration || 300,
|
3451
|
+
i,
|
3452
|
+
path.length
|
3453
|
+
);
|
3454
|
+
await sleep2(delay);
|
3455
|
+
}
|
3456
|
+
}
|
3457
|
+
}
|
3110
3458
|
async waitForNavigation() {
|
3111
3459
|
if (this.pageType === "puppeteer" || this.pageType === "playwright") {
|
3112
3460
|
debugPage("waitForNavigation begin");
|
@@ -3196,7 +3544,8 @@ var Page = class {
|
|
3196
3544
|
return {
|
3197
3545
|
click: async (x, y, options) => {
|
3198
3546
|
await this.mouse.move(x, y);
|
3199
|
-
this.
|
3547
|
+
await this.showClickAnimation(x, y);
|
3548
|
+
await this.underlyingPage.mouse.click(x, y, {
|
3200
3549
|
button: options?.button || "left",
|
3201
3550
|
count: options?.count || 1
|
3202
3551
|
});
|
@@ -3216,27 +3565,23 @@ var Page = class {
|
|
3216
3565
|
},
|
3217
3566
|
move: async (x, y) => {
|
3218
3567
|
this.everMoved = true;
|
3219
|
-
|
3568
|
+
await this.moveMouseHumanLike(x, y);
|
3569
|
+
return Promise.resolve();
|
3220
3570
|
},
|
3221
3571
|
drag: async (from, to) => {
|
3222
3572
|
if (this.pageType === "puppeteer") {
|
3223
|
-
await this.
|
3224
|
-
|
3225
|
-
|
3226
|
-
|
3227
|
-
|
3228
|
-
|
3229
|
-
|
3230
|
-
|
3231
|
-
}
|
3232
|
-
);
|
3573
|
+
await this.mouse.move(from.x, from.y);
|
3574
|
+
await this.underlyingPage.mouse.down();
|
3575
|
+
await this.moveMouseHumanLike(to.x, to.y, {
|
3576
|
+
duration: 500,
|
3577
|
+
// Slower for drag operations
|
3578
|
+
steps: 25
|
3579
|
+
});
|
3580
|
+
await this.underlyingPage.mouse.up();
|
3233
3581
|
} else if (this.pageType === "playwright") {
|
3234
|
-
await this.
|
3235
|
-
from.x,
|
3236
|
-
from.y
|
3237
|
-
);
|
3582
|
+
await this.mouse.move(from.x, from.y);
|
3238
3583
|
await this.underlyingPage.mouse.down();
|
3239
|
-
await this.
|
3584
|
+
await this.mouse.move(to.x, to.y);
|
3240
3585
|
await this.underlyingPage.mouse.up();
|
3241
3586
|
}
|
3242
3587
|
}
|
@@ -3299,6 +3644,18 @@ var Page = class {
|
|
3299
3644
|
await this.mouse.move(targetX, targetY);
|
3300
3645
|
}
|
3301
3646
|
}
|
3647
|
+
/**
|
3648
|
+
* Configure mouse movement behavior
|
3649
|
+
*/
|
3650
|
+
configureMouseMovement(options) {
|
3651
|
+
this.mouseMovementOptions = { ...this.mouseMovementOptions, ...options };
|
3652
|
+
}
|
3653
|
+
/**
|
3654
|
+
* Configure visual feedback
|
3655
|
+
*/
|
3656
|
+
configureVisualFeedback(options) {
|
3657
|
+
this.visualFeedbackOptions = { ...this.visualFeedbackOptions, ...options };
|
3658
|
+
}
|
3302
3659
|
async scrollUntilTop(startingPoint) {
|
3303
3660
|
await this.moveToPointBeforeScroll(startingPoint);
|
3304
3661
|
return this.mouse.wheel(0, -9999999);
|