@projectservan8n/cnapse 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +389 -1
- package/package.json +1 -1
- package/src/components/App.tsx +20 -1
- package/src/services/telegram.ts +120 -0
package/dist/index.js
CHANGED
|
@@ -1250,6 +1250,20 @@ var execAsync3 = promisify3(exec3);
|
|
|
1250
1250
|
import { exec as exec4 } from "child_process";
|
|
1251
1251
|
import { promisify as promisify4 } from "util";
|
|
1252
1252
|
var execAsync4 = promisify4(exec4);
|
|
1253
|
+
async function moveMouse(x, y) {
|
|
1254
|
+
try {
|
|
1255
|
+
if (process.platform === "win32") {
|
|
1256
|
+
await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})"`, { shell: "cmd.exe" });
|
|
1257
|
+
} else if (process.platform === "darwin") {
|
|
1258
|
+
await execAsync4(`cliclick m:${x},${y}`);
|
|
1259
|
+
} else {
|
|
1260
|
+
await execAsync4(`xdotool mousemove ${x} ${y}`);
|
|
1261
|
+
}
|
|
1262
|
+
return ok(`Mouse moved to (${x}, ${y})`);
|
|
1263
|
+
} catch (error) {
|
|
1264
|
+
return err(`Failed to move mouse: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1253
1267
|
async function clickMouse(button = "left") {
|
|
1254
1268
|
try {
|
|
1255
1269
|
if (process.platform === "win32") {
|
|
@@ -1271,6 +1285,28 @@ ${button === "left" ? "[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mous
|
|
|
1271
1285
|
return err(`Failed to click: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1272
1286
|
}
|
|
1273
1287
|
}
|
|
1288
|
+
async function doubleClick() {
|
|
1289
|
+
try {
|
|
1290
|
+
if (process.platform === "win32") {
|
|
1291
|
+
const script = `
|
|
1292
|
+
Add-Type -MemberDefinition @"
|
|
1293
|
+
[DllImport("user32.dll",CharSet=CharSet.Auto,CallingConvention=CallingConvention.StdCall)]
|
|
1294
|
+
public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
|
|
1295
|
+
"@ -Name Mouse -Namespace Win32
|
|
1296
|
+
[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)
|
|
1297
|
+
Start-Sleep -Milliseconds 50
|
|
1298
|
+
[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)`;
|
|
1299
|
+
await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1300
|
+
} else if (process.platform === "darwin") {
|
|
1301
|
+
await execAsync4(`cliclick dc:.`);
|
|
1302
|
+
} else {
|
|
1303
|
+
await execAsync4(`xdotool click --repeat 2 --delay 50 1`);
|
|
1304
|
+
}
|
|
1305
|
+
return ok("Double clicked");
|
|
1306
|
+
} catch (error) {
|
|
1307
|
+
return err(`Failed to double click: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1274
1310
|
async function typeText(text) {
|
|
1275
1311
|
try {
|
|
1276
1312
|
if (process.platform === "win32") {
|
|
@@ -1403,6 +1439,54 @@ async function keyCombo(keys) {
|
|
|
1403
1439
|
return err(`Failed to press combo: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1404
1440
|
}
|
|
1405
1441
|
}
|
|
1442
|
+
async function getActiveWindow() {
|
|
1443
|
+
try {
|
|
1444
|
+
if (process.platform === "win32") {
|
|
1445
|
+
const script = `
|
|
1446
|
+
Add-Type @"
|
|
1447
|
+
using System;
|
|
1448
|
+
using System.Runtime.InteropServices;
|
|
1449
|
+
using System.Text;
|
|
1450
|
+
public class Win32 {
|
|
1451
|
+
[DllImport("user32.dll")]
|
|
1452
|
+
public static extern IntPtr GetForegroundWindow();
|
|
1453
|
+
[DllImport("user32.dll")]
|
|
1454
|
+
public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
|
|
1455
|
+
}
|
|
1456
|
+
"@
|
|
1457
|
+
$hwnd = [Win32]::GetForegroundWindow()
|
|
1458
|
+
$sb = New-Object System.Text.StringBuilder 256
|
|
1459
|
+
[Win32]::GetWindowText($hwnd, $sb, 256)
|
|
1460
|
+
$sb.ToString()`;
|
|
1461
|
+
const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1462
|
+
return ok(stdout.trim() || "Unknown window");
|
|
1463
|
+
} else if (process.platform === "darwin") {
|
|
1464
|
+
const { stdout } = await execAsync4(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
|
|
1465
|
+
return ok(stdout.trim());
|
|
1466
|
+
} else {
|
|
1467
|
+
const { stdout } = await execAsync4(`xdotool getactivewindow getwindowname`);
|
|
1468
|
+
return ok(stdout.trim());
|
|
1469
|
+
}
|
|
1470
|
+
} catch (error) {
|
|
1471
|
+
return err(`Failed to get active window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
async function listWindows() {
|
|
1475
|
+
try {
|
|
1476
|
+
if (process.platform === "win32") {
|
|
1477
|
+
const { stdout } = await execAsync4(`powershell -Command "Get-Process | Where-Object {$_.MainWindowTitle} | Select-Object ProcessName, MainWindowTitle | Format-Table -AutoSize"`, { shell: "cmd.exe" });
|
|
1478
|
+
return ok(stdout);
|
|
1479
|
+
} else if (process.platform === "darwin") {
|
|
1480
|
+
const { stdout } = await execAsync4(`osascript -e 'tell application "System Events" to get name of every application process whose visible is true'`);
|
|
1481
|
+
return ok(stdout);
|
|
1482
|
+
} else {
|
|
1483
|
+
const { stdout } = await execAsync4(`wmctrl -l`);
|
|
1484
|
+
return ok(stdout);
|
|
1485
|
+
}
|
|
1486
|
+
} catch (error) {
|
|
1487
|
+
return err(`Failed to list windows: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1406
1490
|
async function focusWindow(title) {
|
|
1407
1491
|
try {
|
|
1408
1492
|
if (process.platform === "win32") {
|
|
@@ -1418,6 +1502,209 @@ async function focusWindow(title) {
|
|
|
1418
1502
|
return err(`Failed to focus window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1419
1503
|
}
|
|
1420
1504
|
}
|
|
1505
|
+
async function minimizeWindow(title) {
|
|
1506
|
+
try {
|
|
1507
|
+
if (process.platform === "win32") {
|
|
1508
|
+
if (title) {
|
|
1509
|
+
const escaped = title.replace(/'/g, "''");
|
|
1510
|
+
const script = `
|
|
1511
|
+
$proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
1512
|
+
if ($proc) {
|
|
1513
|
+
Add-Type @"
|
|
1514
|
+
using System;
|
|
1515
|
+
using System.Runtime.InteropServices;
|
|
1516
|
+
public class Win32 {
|
|
1517
|
+
[DllImport("user32.dll")]
|
|
1518
|
+
public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
|
1519
|
+
}
|
|
1520
|
+
"@
|
|
1521
|
+
[Win32]::ShowWindow($proc.MainWindowHandle, 6)
|
|
1522
|
+
Write-Output "Minimized: $($proc.MainWindowTitle)"
|
|
1523
|
+
} else {
|
|
1524
|
+
Write-Output "NOT_FOUND"
|
|
1525
|
+
}`;
|
|
1526
|
+
const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1527
|
+
if (stdout.includes("NOT_FOUND")) {
|
|
1528
|
+
return err(`Window containing "${title}" not found`);
|
|
1529
|
+
}
|
|
1530
|
+
return ok(stdout.trim());
|
|
1531
|
+
} else {
|
|
1532
|
+
await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% n')"`, { shell: "cmd.exe" });
|
|
1533
|
+
return ok("Minimized active window");
|
|
1534
|
+
}
|
|
1535
|
+
} else if (process.platform === "darwin") {
|
|
1536
|
+
if (title) {
|
|
1537
|
+
await execAsync4(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to true'`);
|
|
1538
|
+
} else {
|
|
1539
|
+
await execAsync4(`osascript -e 'tell application "System Events" to keystroke "m" using command down'`);
|
|
1540
|
+
}
|
|
1541
|
+
return ok(`Minimized window${title ? `: ${title}` : ""}`);
|
|
1542
|
+
} else {
|
|
1543
|
+
if (title) {
|
|
1544
|
+
await execAsync4(`wmctrl -r "${title}" -b add,hidden`);
|
|
1545
|
+
} else {
|
|
1546
|
+
await execAsync4(`xdotool getactivewindow windowminimize`);
|
|
1547
|
+
}
|
|
1548
|
+
return ok(`Minimized window${title ? `: ${title}` : ""}`);
|
|
1549
|
+
}
|
|
1550
|
+
} catch (error) {
|
|
1551
|
+
return err(`Failed to minimize window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
async function maximizeWindow(title) {
|
|
1555
|
+
try {
|
|
1556
|
+
if (process.platform === "win32") {
|
|
1557
|
+
if (title) {
|
|
1558
|
+
const escaped = title.replace(/'/g, "''");
|
|
1559
|
+
const script = `
|
|
1560
|
+
$proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
1561
|
+
if ($proc) {
|
|
1562
|
+
Add-Type @"
|
|
1563
|
+
using System;
|
|
1564
|
+
using System.Runtime.InteropServices;
|
|
1565
|
+
public class Win32 {
|
|
1566
|
+
[DllImport("user32.dll")]
|
|
1567
|
+
public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
|
1568
|
+
}
|
|
1569
|
+
"@
|
|
1570
|
+
[Win32]::ShowWindow($proc.MainWindowHandle, 3)
|
|
1571
|
+
Write-Output "Maximized: $($proc.MainWindowTitle)"
|
|
1572
|
+
} else {
|
|
1573
|
+
Write-Output "NOT_FOUND"
|
|
1574
|
+
}`;
|
|
1575
|
+
const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1576
|
+
if (stdout.includes("NOT_FOUND")) {
|
|
1577
|
+
return err(`Window containing "${title}" not found`);
|
|
1578
|
+
}
|
|
1579
|
+
return ok(stdout.trim());
|
|
1580
|
+
} else {
|
|
1581
|
+
await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% x')"`, { shell: "cmd.exe" });
|
|
1582
|
+
return ok("Maximized active window");
|
|
1583
|
+
}
|
|
1584
|
+
} else if (process.platform === "darwin") {
|
|
1585
|
+
if (title) {
|
|
1586
|
+
await execAsync4(`osascript -e 'tell application "${title}" to set zoomed of window 1 to true'`);
|
|
1587
|
+
} else {
|
|
1588
|
+
await execAsync4(`osascript -e 'tell application "System Events" to keystroke "f" using {control down, command down}'`);
|
|
1589
|
+
}
|
|
1590
|
+
return ok(`Maximized window${title ? `: ${title}` : ""}`);
|
|
1591
|
+
} else {
|
|
1592
|
+
if (title) {
|
|
1593
|
+
await execAsync4(`wmctrl -r "${title}" -b add,maximized_vert,maximized_horz`);
|
|
1594
|
+
} else {
|
|
1595
|
+
await execAsync4(`wmctrl -r :ACTIVE: -b add,maximized_vert,maximized_horz`);
|
|
1596
|
+
}
|
|
1597
|
+
return ok(`Maximized window${title ? `: ${title}` : ""}`);
|
|
1598
|
+
}
|
|
1599
|
+
} catch (error) {
|
|
1600
|
+
return err(`Failed to maximize window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
async function closeWindow(title) {
|
|
1604
|
+
try {
|
|
1605
|
+
if (process.platform === "win32") {
|
|
1606
|
+
if (title) {
|
|
1607
|
+
const escaped = title.replace(/'/g, "''");
|
|
1608
|
+
await execAsync4(`powershell -Command "Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' } | ForEach-Object { $_.CloseMainWindow() }"`, { shell: "cmd.exe" });
|
|
1609
|
+
return ok(`Closed window: ${title}`);
|
|
1610
|
+
} else {
|
|
1611
|
+
await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('%{F4}')"`, { shell: "cmd.exe" });
|
|
1612
|
+
return ok("Closed active window");
|
|
1613
|
+
}
|
|
1614
|
+
} else if (process.platform === "darwin") {
|
|
1615
|
+
if (title) {
|
|
1616
|
+
await execAsync4(`osascript -e 'tell application "${title}" to close window 1'`);
|
|
1617
|
+
} else {
|
|
1618
|
+
await execAsync4(`osascript -e 'tell application "System Events" to keystroke "w" using command down'`);
|
|
1619
|
+
}
|
|
1620
|
+
return ok(`Closed window${title ? `: ${title}` : ""}`);
|
|
1621
|
+
} else {
|
|
1622
|
+
if (title) {
|
|
1623
|
+
await execAsync4(`wmctrl -c "${title}"`);
|
|
1624
|
+
} else {
|
|
1625
|
+
await execAsync4(`xdotool getactivewindow windowclose`);
|
|
1626
|
+
}
|
|
1627
|
+
return ok(`Closed window${title ? `: ${title}` : ""}`);
|
|
1628
|
+
}
|
|
1629
|
+
} catch (error) {
|
|
1630
|
+
return err(`Failed to close window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
async function restoreWindow(title) {
|
|
1634
|
+
try {
|
|
1635
|
+
if (process.platform === "win32") {
|
|
1636
|
+
const escaped = title.replace(/'/g, "''");
|
|
1637
|
+
const script = `
|
|
1638
|
+
$proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
1639
|
+
if ($proc) {
|
|
1640
|
+
Add-Type @"
|
|
1641
|
+
using System;
|
|
1642
|
+
using System.Runtime.InteropServices;
|
|
1643
|
+
public class Win32 {
|
|
1644
|
+
[DllImport("user32.dll")]
|
|
1645
|
+
public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
|
1646
|
+
}
|
|
1647
|
+
"@
|
|
1648
|
+
[Win32]::ShowWindow($proc.MainWindowHandle, 9)
|
|
1649
|
+
Write-Output "Restored: $($proc.MainWindowTitle)"
|
|
1650
|
+
} else {
|
|
1651
|
+
Write-Output "NOT_FOUND"
|
|
1652
|
+
}`;
|
|
1653
|
+
const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1654
|
+
if (stdout.includes("NOT_FOUND")) {
|
|
1655
|
+
return err(`Window containing "${title}" not found`);
|
|
1656
|
+
}
|
|
1657
|
+
return ok(stdout.trim());
|
|
1658
|
+
} else if (process.platform === "darwin") {
|
|
1659
|
+
await execAsync4(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to false'`);
|
|
1660
|
+
return ok(`Restored window: ${title}`);
|
|
1661
|
+
} else {
|
|
1662
|
+
await execAsync4(`wmctrl -r "${title}" -b remove,hidden`);
|
|
1663
|
+
return ok(`Restored window: ${title}`);
|
|
1664
|
+
}
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
return err(`Failed to restore window: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
async function scrollMouse(amount) {
|
|
1670
|
+
try {
|
|
1671
|
+
if (process.platform === "win32") {
|
|
1672
|
+
const direction = amount > 0 ? 120 * Math.abs(amount) : -120 * Math.abs(amount);
|
|
1673
|
+
const script = `
|
|
1674
|
+
Add-Type -MemberDefinition @"
|
|
1675
|
+
[DllImport("user32.dll",CharSet=CharSet.Auto,CallingConvention=CallingConvention.StdCall)]
|
|
1676
|
+
public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
|
|
1677
|
+
"@ -Name Mouse -Namespace Win32
|
|
1678
|
+
[Win32.Mouse]::mouse_event(0x0800, 0, 0, ${direction}, 0)`;
|
|
1679
|
+
await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1680
|
+
} else if (process.platform === "darwin") {
|
|
1681
|
+
const dir = amount > 0 ? "u" : "d";
|
|
1682
|
+
await execAsync4(`cliclick -r ${dir}:${Math.abs(amount)}`);
|
|
1683
|
+
} else {
|
|
1684
|
+
const btn = amount > 0 ? "4" : "5";
|
|
1685
|
+
await execAsync4(`xdotool click --repeat ${Math.abs(amount)} ${btn}`);
|
|
1686
|
+
}
|
|
1687
|
+
return ok(`Scrolled ${amount > 0 ? "up" : "down"} by ${Math.abs(amount)}`);
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
return err(`Failed to scroll: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
async function getMousePosition() {
|
|
1693
|
+
try {
|
|
1694
|
+
if (process.platform === "win32") {
|
|
1695
|
+
const { stdout } = await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; $p = [System.Windows.Forms.Cursor]::Position; Write-Output \\"$($p.X),$($p.Y)\\""`, { shell: "cmd.exe" });
|
|
1696
|
+
return ok(`Mouse position: ${stdout.trim()}`);
|
|
1697
|
+
} else if (process.platform === "darwin") {
|
|
1698
|
+
const { stdout } = await execAsync4(`cliclick p`);
|
|
1699
|
+
return ok(`Mouse position: ${stdout.trim()}`);
|
|
1700
|
+
} else {
|
|
1701
|
+
const { stdout } = await execAsync4(`xdotool getmouselocation --shell`);
|
|
1702
|
+
return ok(stdout);
|
|
1703
|
+
}
|
|
1704
|
+
} catch (error) {
|
|
1705
|
+
return err(`Failed to get mouse position: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1421
1708
|
|
|
1422
1709
|
// src/tools/index.ts
|
|
1423
1710
|
function ok(output) {
|
|
@@ -1642,6 +1929,12 @@ ${result.error}
|
|
|
1642
1929
|
}
|
|
1643
1930
|
try {
|
|
1644
1931
|
await ctx.sendChatAction("typing");
|
|
1932
|
+
const computerControlResult = await this.tryComputerControl(userText);
|
|
1933
|
+
if (computerControlResult) {
|
|
1934
|
+
await ctx.reply(computerControlResult);
|
|
1935
|
+
history.push({ role: "assistant", content: computerControlResult });
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1645
1938
|
const isVisionRequest = /screen|see|look|what('?s| is) (on|visible)|show me|screenshot/i.test(userText);
|
|
1646
1939
|
let response;
|
|
1647
1940
|
if (isVisionRequest) {
|
|
@@ -1683,6 +1976,85 @@ ${result.error}
|
|
|
1683
1976
|
}
|
|
1684
1977
|
return this.allowedChatIds.has(chatId);
|
|
1685
1978
|
}
|
|
1979
|
+
/**
|
|
1980
|
+
* Try to execute computer control commands directly
|
|
1981
|
+
* Returns response string if handled, null if not a computer command
|
|
1982
|
+
*/
|
|
1983
|
+
async tryComputerControl(text) {
|
|
1984
|
+
const lower = text.toLowerCase();
|
|
1985
|
+
let match = lower.match(/minimize\s+(?:the\s+)?(.+)/i);
|
|
1986
|
+
if (match) {
|
|
1987
|
+
const result = await minimizeWindow(match[1].trim());
|
|
1988
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
1989
|
+
}
|
|
1990
|
+
match = lower.match(/maximize\s+(?:the\s+)?(.+)/i);
|
|
1991
|
+
if (match) {
|
|
1992
|
+
const result = await maximizeWindow(match[1].trim());
|
|
1993
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
1994
|
+
}
|
|
1995
|
+
match = lower.match(/close\s+(?:the\s+)?(.+)/i);
|
|
1996
|
+
if (match) {
|
|
1997
|
+
const result = await closeWindow(match[1].trim());
|
|
1998
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
1999
|
+
}
|
|
2000
|
+
match = lower.match(/restore\s+(?:the\s+)?(.+)/i);
|
|
2001
|
+
if (match) {
|
|
2002
|
+
const result = await restoreWindow(match[1].trim());
|
|
2003
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2004
|
+
}
|
|
2005
|
+
match = lower.match(/(?:focus|open|switch to)\s+(?:the\s+)?(.+)/i);
|
|
2006
|
+
if (match) {
|
|
2007
|
+
const result = await focusWindow(match[1].trim());
|
|
2008
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2009
|
+
}
|
|
2010
|
+
match = text.match(/type\s+["'](.+)["']/i);
|
|
2011
|
+
if (match) {
|
|
2012
|
+
const result = await typeText(match[1]);
|
|
2013
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2014
|
+
}
|
|
2015
|
+
match = lower.match(/press\s+(?:the\s+)?(\w+)/i);
|
|
2016
|
+
if (match) {
|
|
2017
|
+
const result = await pressKey(match[1]);
|
|
2018
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2019
|
+
}
|
|
2020
|
+
if (/^click$/i.test(lower) || /click\s+(?:the\s+)?mouse/i.test(lower)) {
|
|
2021
|
+
const result = await clickMouse("left");
|
|
2022
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2023
|
+
}
|
|
2024
|
+
if (/right\s*click/i.test(lower)) {
|
|
2025
|
+
const result = await clickMouse("right");
|
|
2026
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2027
|
+
}
|
|
2028
|
+
if (/double\s*click/i.test(lower)) {
|
|
2029
|
+
const result = await doubleClick();
|
|
2030
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2031
|
+
}
|
|
2032
|
+
match = lower.match(/move\s+(?:the\s+)?mouse\s+(?:to\s+)?(\d+)[,\s]+(\d+)/i);
|
|
2033
|
+
if (match) {
|
|
2034
|
+
const result = await moveMouse(parseInt(match[1]), parseInt(match[2]));
|
|
2035
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2036
|
+
}
|
|
2037
|
+
match = lower.match(/scroll\s+(up|down)(?:\s+(\d+))?/i);
|
|
2038
|
+
if (match) {
|
|
2039
|
+
const amount = match[1] === "up" ? parseInt(match[2]) || 3 : -(parseInt(match[2]) || 3);
|
|
2040
|
+
const result = await scrollMouse(amount);
|
|
2041
|
+
return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
|
|
2042
|
+
}
|
|
2043
|
+
if (/list\s+(?:all\s+)?windows/i.test(lower) || /what\s+windows/i.test(lower)) {
|
|
2044
|
+
const result = await listWindows();
|
|
2045
|
+
return result.success ? `\u{1F4CB} Open Windows:
|
|
2046
|
+
${result.output}` : `\u274C ${result.error}`;
|
|
2047
|
+
}
|
|
2048
|
+
if (/(?:active|current|focused)\s+window/i.test(lower) || /what\s+(?:window|app)/i.test(lower)) {
|
|
2049
|
+
const result = await getActiveWindow();
|
|
2050
|
+
return result.success ? `\u{1FA9F} Active: ${result.output}` : `\u274C ${result.error}`;
|
|
2051
|
+
}
|
|
2052
|
+
if (/mouse\s+position/i.test(lower) || /where.*mouse/i.test(lower)) {
|
|
2053
|
+
const result = await getMousePosition();
|
|
2054
|
+
return result.success ? `\u{1F5B1}\uFE0F ${result.output}` : `\u274C ${result.error}`;
|
|
2055
|
+
}
|
|
2056
|
+
return null;
|
|
2057
|
+
}
|
|
1686
2058
|
/**
|
|
1687
2059
|
* Send a message to a specific chat
|
|
1688
2060
|
*/
|
|
@@ -2311,17 +2683,33 @@ ${tasks.format(task)}`);
|
|
|
2311
2683
|
setStatus("Ready");
|
|
2312
2684
|
}
|
|
2313
2685
|
}, [chat2, tasks]);
|
|
2686
|
+
const isComputerControlRequest = useCallback5((text) => {
|
|
2687
|
+
const lower = text.toLowerCase();
|
|
2688
|
+
const patterns = [
|
|
2689
|
+
/^(can you |please |)?(open|close|minimize|maximize|restore|focus|click|type|press|scroll|move|drag)/i,
|
|
2690
|
+
/^(can you |please |)?move (the |my |)mouse/i,
|
|
2691
|
+
/^(can you |please |)?(start|launch|run) [a-z]/i,
|
|
2692
|
+
/(open|close|minimize|maximize) (the |my |)?[a-z]/i,
|
|
2693
|
+
/click (on |the |)/i,
|
|
2694
|
+
/type ["'].+["']/i,
|
|
2695
|
+
/press (enter|escape|tab|ctrl|alt|shift|space|backspace|delete|f\d+)/i
|
|
2696
|
+
];
|
|
2697
|
+
return patterns.some((p) => p.test(lower));
|
|
2698
|
+
}, []);
|
|
2314
2699
|
const handleSubmit = useCallback5(async (value) => {
|
|
2315
2700
|
if (!value.trim()) return;
|
|
2316
2701
|
setInputValue("");
|
|
2317
2702
|
if (value.startsWith("/")) {
|
|
2318
2703
|
await handleCommand(value);
|
|
2704
|
+
} else if (isComputerControlRequest(value)) {
|
|
2705
|
+
chat2.addSystemMessage(`\u{1F916} Executing: ${value}`);
|
|
2706
|
+
await handleTaskCommand(value);
|
|
2319
2707
|
} else {
|
|
2320
2708
|
setStatus("Thinking...");
|
|
2321
2709
|
await chat2.sendMessage(value);
|
|
2322
2710
|
setStatus("Ready");
|
|
2323
2711
|
}
|
|
2324
|
-
}, [chat2, handleCommand]);
|
|
2712
|
+
}, [chat2, handleCommand, handleTaskCommand, isComputerControlRequest]);
|
|
2325
2713
|
const handleProviderSelect = useCallback5((provider, model) => {
|
|
2326
2714
|
chat2.addSystemMessage(`\u2705 Updated: ${provider} / ${model}`);
|
|
2327
2715
|
}, [chat2]);
|
package/package.json
CHANGED
package/src/components/App.tsx
CHANGED
|
@@ -202,6 +202,21 @@ export function App() {
|
|
|
202
202
|
}
|
|
203
203
|
}, [chat, tasks]);
|
|
204
204
|
|
|
205
|
+
// Check if message looks like a computer control request
|
|
206
|
+
const isComputerControlRequest = useCallback((text: string): boolean => {
|
|
207
|
+
const lower = text.toLowerCase();
|
|
208
|
+
const patterns = [
|
|
209
|
+
/^(can you |please |)?(open|close|minimize|maximize|restore|focus|click|type|press|scroll|move|drag)/i,
|
|
210
|
+
/^(can you |please |)?move (the |my |)mouse/i,
|
|
211
|
+
/^(can you |please |)?(start|launch|run) [a-z]/i,
|
|
212
|
+
/(open|close|minimize|maximize) (the |my |)?[a-z]/i,
|
|
213
|
+
/click (on |the |)/i,
|
|
214
|
+
/type ["'].+["']/i,
|
|
215
|
+
/press (enter|escape|tab|ctrl|alt|shift|space|backspace|delete|f\d+)/i,
|
|
216
|
+
];
|
|
217
|
+
return patterns.some(p => p.test(lower));
|
|
218
|
+
}, []);
|
|
219
|
+
|
|
205
220
|
// Submit handler
|
|
206
221
|
const handleSubmit = useCallback(async (value: string) => {
|
|
207
222
|
if (!value.trim()) return;
|
|
@@ -209,12 +224,16 @@ export function App() {
|
|
|
209
224
|
|
|
210
225
|
if (value.startsWith('/')) {
|
|
211
226
|
await handleCommand(value);
|
|
227
|
+
} else if (isComputerControlRequest(value)) {
|
|
228
|
+
// Auto-route to task system for computer control
|
|
229
|
+
chat.addSystemMessage(`🤖 Executing: ${value}`);
|
|
230
|
+
await handleTaskCommand(value);
|
|
212
231
|
} else {
|
|
213
232
|
setStatus('Thinking...');
|
|
214
233
|
await chat.sendMessage(value);
|
|
215
234
|
setStatus('Ready');
|
|
216
235
|
}
|
|
217
|
-
}, [chat, handleCommand]);
|
|
236
|
+
}, [chat, handleCommand, handleTaskCommand, isComputerControlRequest]);
|
|
218
237
|
|
|
219
238
|
// Provider selection callback
|
|
220
239
|
const handleProviderSelect = useCallback((provider: string, model: string) => {
|
package/src/services/telegram.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { getConfig, getApiKey } from '../lib/config.js';
|
|
|
7
7
|
import { describeScreen, captureScreenshot } from '../lib/vision.js';
|
|
8
8
|
import { runCommand } from '../tools/shell.js';
|
|
9
9
|
import { chat as chatWithAI, chatWithVision, Message } from '../lib/api.js';
|
|
10
|
+
import * as computer from '../tools/computer.js';
|
|
10
11
|
|
|
11
12
|
export interface TelegramMessage {
|
|
12
13
|
chatId: number;
|
|
@@ -248,6 +249,14 @@ export class TelegramBotService extends EventEmitter {
|
|
|
248
249
|
// Send typing indicator
|
|
249
250
|
await ctx.sendChatAction('typing');
|
|
250
251
|
|
|
252
|
+
// Check if this looks like a computer control request
|
|
253
|
+
const computerControlResult = await this.tryComputerControl(userText);
|
|
254
|
+
if (computerControlResult) {
|
|
255
|
+
await ctx.reply(computerControlResult);
|
|
256
|
+
history.push({ role: 'assistant', content: computerControlResult });
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
|
|
251
260
|
// Check if this looks like a screen/vision request
|
|
252
261
|
const isVisionRequest = /screen|see|look|what('?s| is) (on|visible)|show me|screenshot/i.test(userText);
|
|
253
262
|
|
|
@@ -302,6 +311,117 @@ export class TelegramBotService extends EventEmitter {
|
|
|
302
311
|
return this.allowedChatIds.has(chatId);
|
|
303
312
|
}
|
|
304
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Try to execute computer control commands directly
|
|
316
|
+
* Returns response string if handled, null if not a computer command
|
|
317
|
+
*/
|
|
318
|
+
private async tryComputerControl(text: string): Promise<string | null> {
|
|
319
|
+
const lower = text.toLowerCase();
|
|
320
|
+
|
|
321
|
+
// Minimize window
|
|
322
|
+
let match = lower.match(/minimize\s+(?:the\s+)?(.+)/i);
|
|
323
|
+
if (match) {
|
|
324
|
+
const result = await computer.minimizeWindow(match[1].trim());
|
|
325
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Maximize window
|
|
329
|
+
match = lower.match(/maximize\s+(?:the\s+)?(.+)/i);
|
|
330
|
+
if (match) {
|
|
331
|
+
const result = await computer.maximizeWindow(match[1].trim());
|
|
332
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// Close window
|
|
336
|
+
match = lower.match(/close\s+(?:the\s+)?(.+)/i);
|
|
337
|
+
if (match) {
|
|
338
|
+
const result = await computer.closeWindow(match[1].trim());
|
|
339
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// Restore window
|
|
343
|
+
match = lower.match(/restore\s+(?:the\s+)?(.+)/i);
|
|
344
|
+
if (match) {
|
|
345
|
+
const result = await computer.restoreWindow(match[1].trim());
|
|
346
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Focus/open window
|
|
350
|
+
match = lower.match(/(?:focus|open|switch to)\s+(?:the\s+)?(.+)/i);
|
|
351
|
+
if (match) {
|
|
352
|
+
const result = await computer.focusWindow(match[1].trim());
|
|
353
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Type text
|
|
357
|
+
match = text.match(/type\s+["'](.+)["']/i);
|
|
358
|
+
if (match) {
|
|
359
|
+
const result = await computer.typeText(match[1]);
|
|
360
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Press key
|
|
364
|
+
match = lower.match(/press\s+(?:the\s+)?(\w+)/i);
|
|
365
|
+
if (match) {
|
|
366
|
+
const result = await computer.pressKey(match[1]);
|
|
367
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// Click
|
|
371
|
+
if (/^click$/i.test(lower) || /click\s+(?:the\s+)?mouse/i.test(lower)) {
|
|
372
|
+
const result = await computer.clickMouse('left');
|
|
373
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Right click
|
|
377
|
+
if (/right\s*click/i.test(lower)) {
|
|
378
|
+
const result = await computer.clickMouse('right');
|
|
379
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Double click
|
|
383
|
+
if (/double\s*click/i.test(lower)) {
|
|
384
|
+
const result = await computer.doubleClick();
|
|
385
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Move mouse to coordinates
|
|
389
|
+
match = lower.match(/move\s+(?:the\s+)?mouse\s+(?:to\s+)?(\d+)[,\s]+(\d+)/i);
|
|
390
|
+
if (match) {
|
|
391
|
+
const result = await computer.moveMouse(parseInt(match[1]), parseInt(match[2]));
|
|
392
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// Scroll
|
|
396
|
+
match = lower.match(/scroll\s+(up|down)(?:\s+(\d+))?/i);
|
|
397
|
+
if (match) {
|
|
398
|
+
const amount = match[1] === 'up' ? (parseInt(match[2]) || 3) : -(parseInt(match[2]) || 3);
|
|
399
|
+
const result = await computer.scrollMouse(amount);
|
|
400
|
+
return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// List windows
|
|
404
|
+
if (/list\s+(?:all\s+)?windows/i.test(lower) || /what\s+windows/i.test(lower)) {
|
|
405
|
+
const result = await computer.listWindows();
|
|
406
|
+
return result.success ? `📋 Open Windows:\n${result.output}` : `❌ ${result.error}`;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// Get active window
|
|
410
|
+
if (/(?:active|current|focused)\s+window/i.test(lower) || /what\s+(?:window|app)/i.test(lower)) {
|
|
411
|
+
const result = await computer.getActiveWindow();
|
|
412
|
+
return result.success ? `🪟 Active: ${result.output}` : `❌ ${result.error}`;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Mouse position
|
|
416
|
+
if (/mouse\s+position/i.test(lower) || /where.*mouse/i.test(lower)) {
|
|
417
|
+
const result = await computer.getMousePosition();
|
|
418
|
+
return result.success ? `🖱️ ${result.output}` : `❌ ${result.error}`;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// Not a computer control command
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
|
|
305
425
|
/**
|
|
306
426
|
* Send a message to a specific chat
|
|
307
427
|
*/
|