@projectservan8n/cnapse 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1250,6 +1250,20 @@ var execAsync3 = promisify3(exec3);
1250
1250
  import { exec as exec4 } from "child_process";
1251
1251
  import { promisify as promisify4 } from "util";
1252
1252
  var execAsync4 = promisify4(exec4);
1253
+ async function moveMouse(x, y) {
1254
+ try {
1255
+ if (process.platform === "win32") {
1256
+ await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})"`, { shell: "cmd.exe" });
1257
+ } else if (process.platform === "darwin") {
1258
+ await execAsync4(`cliclick m:${x},${y}`);
1259
+ } else {
1260
+ await execAsync4(`xdotool mousemove ${x} ${y}`);
1261
+ }
1262
+ return ok(`Mouse moved to (${x}, ${y})`);
1263
+ } catch (error) {
1264
+ return err(`Failed to move mouse: ${error instanceof Error ? error.message : "Unknown error"}`);
1265
+ }
1266
+ }
1253
1267
  async function clickMouse(button = "left") {
1254
1268
  try {
1255
1269
  if (process.platform === "win32") {
@@ -1271,6 +1285,28 @@ ${button === "left" ? "[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mous
1271
1285
  return err(`Failed to click: ${error instanceof Error ? error.message : "Unknown error"}`);
1272
1286
  }
1273
1287
  }
1288
+ async function doubleClick() {
1289
+ try {
1290
+ if (process.platform === "win32") {
1291
+ const script = `
1292
+ Add-Type -MemberDefinition @"
1293
+ [DllImport("user32.dll",CharSet=CharSet.Auto,CallingConvention=CallingConvention.StdCall)]
1294
+ public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
1295
+ "@ -Name Mouse -Namespace Win32
1296
+ [Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)
1297
+ Start-Sleep -Milliseconds 50
1298
+ [Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)`;
1299
+ await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1300
+ } else if (process.platform === "darwin") {
1301
+ await execAsync4(`cliclick dc:.`);
1302
+ } else {
1303
+ await execAsync4(`xdotool click --repeat 2 --delay 50 1`);
1304
+ }
1305
+ return ok("Double clicked");
1306
+ } catch (error) {
1307
+ return err(`Failed to double click: ${error instanceof Error ? error.message : "Unknown error"}`);
1308
+ }
1309
+ }
1274
1310
  async function typeText(text) {
1275
1311
  try {
1276
1312
  if (process.platform === "win32") {
@@ -1403,6 +1439,54 @@ async function keyCombo(keys) {
1403
1439
  return err(`Failed to press combo: ${error instanceof Error ? error.message : "Unknown error"}`);
1404
1440
  }
1405
1441
  }
1442
+ async function getActiveWindow() {
1443
+ try {
1444
+ if (process.platform === "win32") {
1445
+ const script = `
1446
+ Add-Type @"
1447
+ using System;
1448
+ using System.Runtime.InteropServices;
1449
+ using System.Text;
1450
+ public class Win32 {
1451
+ [DllImport("user32.dll")]
1452
+ public static extern IntPtr GetForegroundWindow();
1453
+ [DllImport("user32.dll")]
1454
+ public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
1455
+ }
1456
+ "@
1457
+ $hwnd = [Win32]::GetForegroundWindow()
1458
+ $sb = New-Object System.Text.StringBuilder 256
1459
+ [Win32]::GetWindowText($hwnd, $sb, 256)
1460
+ $sb.ToString()`;
1461
+ const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1462
+ return ok(stdout.trim() || "Unknown window");
1463
+ } else if (process.platform === "darwin") {
1464
+ const { stdout } = await execAsync4(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
1465
+ return ok(stdout.trim());
1466
+ } else {
1467
+ const { stdout } = await execAsync4(`xdotool getactivewindow getwindowname`);
1468
+ return ok(stdout.trim());
1469
+ }
1470
+ } catch (error) {
1471
+ return err(`Failed to get active window: ${error instanceof Error ? error.message : "Unknown error"}`);
1472
+ }
1473
+ }
1474
+ async function listWindows() {
1475
+ try {
1476
+ if (process.platform === "win32") {
1477
+ const { stdout } = await execAsync4(`powershell -Command "Get-Process | Where-Object {$_.MainWindowTitle} | Select-Object ProcessName, MainWindowTitle | Format-Table -AutoSize"`, { shell: "cmd.exe" });
1478
+ return ok(stdout);
1479
+ } else if (process.platform === "darwin") {
1480
+ const { stdout } = await execAsync4(`osascript -e 'tell application "System Events" to get name of every application process whose visible is true'`);
1481
+ return ok(stdout);
1482
+ } else {
1483
+ const { stdout } = await execAsync4(`wmctrl -l`);
1484
+ return ok(stdout);
1485
+ }
1486
+ } catch (error) {
1487
+ return err(`Failed to list windows: ${error instanceof Error ? error.message : "Unknown error"}`);
1488
+ }
1489
+ }
1406
1490
  async function focusWindow(title) {
1407
1491
  try {
1408
1492
  if (process.platform === "win32") {
@@ -1418,6 +1502,209 @@ async function focusWindow(title) {
1418
1502
  return err(`Failed to focus window: ${error instanceof Error ? error.message : "Unknown error"}`);
1419
1503
  }
1420
1504
  }
1505
+ async function minimizeWindow(title) {
1506
+ try {
1507
+ if (process.platform === "win32") {
1508
+ if (title) {
1509
+ const escaped = title.replace(/'/g, "''");
1510
+ const script = `
1511
+ $proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
1512
+ if ($proc) {
1513
+ Add-Type @"
1514
+ using System;
1515
+ using System.Runtime.InteropServices;
1516
+ public class Win32 {
1517
+ [DllImport("user32.dll")]
1518
+ public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
1519
+ }
1520
+ "@
1521
+ [Win32]::ShowWindow($proc.MainWindowHandle, 6)
1522
+ Write-Output "Minimized: $($proc.MainWindowTitle)"
1523
+ } else {
1524
+ Write-Output "NOT_FOUND"
1525
+ }`;
1526
+ const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1527
+ if (stdout.includes("NOT_FOUND")) {
1528
+ return err(`Window containing "${title}" not found`);
1529
+ }
1530
+ return ok(stdout.trim());
1531
+ } else {
1532
+ await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% n')"`, { shell: "cmd.exe" });
1533
+ return ok("Minimized active window");
1534
+ }
1535
+ } else if (process.platform === "darwin") {
1536
+ if (title) {
1537
+ await execAsync4(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to true'`);
1538
+ } else {
1539
+ await execAsync4(`osascript -e 'tell application "System Events" to keystroke "m" using command down'`);
1540
+ }
1541
+ return ok(`Minimized window${title ? `: ${title}` : ""}`);
1542
+ } else {
1543
+ if (title) {
1544
+ await execAsync4(`wmctrl -r "${title}" -b add,hidden`);
1545
+ } else {
1546
+ await execAsync4(`xdotool getactivewindow windowminimize`);
1547
+ }
1548
+ return ok(`Minimized window${title ? `: ${title}` : ""}`);
1549
+ }
1550
+ } catch (error) {
1551
+ return err(`Failed to minimize window: ${error instanceof Error ? error.message : "Unknown error"}`);
1552
+ }
1553
+ }
1554
+ async function maximizeWindow(title) {
1555
+ try {
1556
+ if (process.platform === "win32") {
1557
+ if (title) {
1558
+ const escaped = title.replace(/'/g, "''");
1559
+ const script = `
1560
+ $proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
1561
+ if ($proc) {
1562
+ Add-Type @"
1563
+ using System;
1564
+ using System.Runtime.InteropServices;
1565
+ public class Win32 {
1566
+ [DllImport("user32.dll")]
1567
+ public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
1568
+ }
1569
+ "@
1570
+ [Win32]::ShowWindow($proc.MainWindowHandle, 3)
1571
+ Write-Output "Maximized: $($proc.MainWindowTitle)"
1572
+ } else {
1573
+ Write-Output "NOT_FOUND"
1574
+ }`;
1575
+ const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1576
+ if (stdout.includes("NOT_FOUND")) {
1577
+ return err(`Window containing "${title}" not found`);
1578
+ }
1579
+ return ok(stdout.trim());
1580
+ } else {
1581
+ await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% x')"`, { shell: "cmd.exe" });
1582
+ return ok("Maximized active window");
1583
+ }
1584
+ } else if (process.platform === "darwin") {
1585
+ if (title) {
1586
+ await execAsync4(`osascript -e 'tell application "${title}" to set zoomed of window 1 to true'`);
1587
+ } else {
1588
+ await execAsync4(`osascript -e 'tell application "System Events" to keystroke "f" using {control down, command down}'`);
1589
+ }
1590
+ return ok(`Maximized window${title ? `: ${title}` : ""}`);
1591
+ } else {
1592
+ if (title) {
1593
+ await execAsync4(`wmctrl -r "${title}" -b add,maximized_vert,maximized_horz`);
1594
+ } else {
1595
+ await execAsync4(`wmctrl -r :ACTIVE: -b add,maximized_vert,maximized_horz`);
1596
+ }
1597
+ return ok(`Maximized window${title ? `: ${title}` : ""}`);
1598
+ }
1599
+ } catch (error) {
1600
+ return err(`Failed to maximize window: ${error instanceof Error ? error.message : "Unknown error"}`);
1601
+ }
1602
+ }
1603
+ async function closeWindow(title) {
1604
+ try {
1605
+ if (process.platform === "win32") {
1606
+ if (title) {
1607
+ const escaped = title.replace(/'/g, "''");
1608
+ await execAsync4(`powershell -Command "Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' } | ForEach-Object { $_.CloseMainWindow() }"`, { shell: "cmd.exe" });
1609
+ return ok(`Closed window: ${title}`);
1610
+ } else {
1611
+ await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('%{F4}')"`, { shell: "cmd.exe" });
1612
+ return ok("Closed active window");
1613
+ }
1614
+ } else if (process.platform === "darwin") {
1615
+ if (title) {
1616
+ await execAsync4(`osascript -e 'tell application "${title}" to close window 1'`);
1617
+ } else {
1618
+ await execAsync4(`osascript -e 'tell application "System Events" to keystroke "w" using command down'`);
1619
+ }
1620
+ return ok(`Closed window${title ? `: ${title}` : ""}`);
1621
+ } else {
1622
+ if (title) {
1623
+ await execAsync4(`wmctrl -c "${title}"`);
1624
+ } else {
1625
+ await execAsync4(`xdotool getactivewindow windowclose`);
1626
+ }
1627
+ return ok(`Closed window${title ? `: ${title}` : ""}`);
1628
+ }
1629
+ } catch (error) {
1630
+ return err(`Failed to close window: ${error instanceof Error ? error.message : "Unknown error"}`);
1631
+ }
1632
+ }
1633
+ async function restoreWindow(title) {
1634
+ try {
1635
+ if (process.platform === "win32") {
1636
+ const escaped = title.replace(/'/g, "''");
1637
+ const script = `
1638
+ $proc = Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
1639
+ if ($proc) {
1640
+ Add-Type @"
1641
+ using System;
1642
+ using System.Runtime.InteropServices;
1643
+ public class Win32 {
1644
+ [DllImport("user32.dll")]
1645
+ public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
1646
+ }
1647
+ "@
1648
+ [Win32]::ShowWindow($proc.MainWindowHandle, 9)
1649
+ Write-Output "Restored: $($proc.MainWindowTitle)"
1650
+ } else {
1651
+ Write-Output "NOT_FOUND"
1652
+ }`;
1653
+ const { stdout } = await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1654
+ if (stdout.includes("NOT_FOUND")) {
1655
+ return err(`Window containing "${title}" not found`);
1656
+ }
1657
+ return ok(stdout.trim());
1658
+ } else if (process.platform === "darwin") {
1659
+ await execAsync4(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to false'`);
1660
+ return ok(`Restored window: ${title}`);
1661
+ } else {
1662
+ await execAsync4(`wmctrl -r "${title}" -b remove,hidden`);
1663
+ return ok(`Restored window: ${title}`);
1664
+ }
1665
+ } catch (error) {
1666
+ return err(`Failed to restore window: ${error instanceof Error ? error.message : "Unknown error"}`);
1667
+ }
1668
+ }
1669
+ async function scrollMouse(amount) {
1670
+ try {
1671
+ if (process.platform === "win32") {
1672
+ const direction = amount > 0 ? 120 * Math.abs(amount) : -120 * Math.abs(amount);
1673
+ const script = `
1674
+ Add-Type -MemberDefinition @"
1675
+ [DllImport("user32.dll",CharSet=CharSet.Auto,CallingConvention=CallingConvention.StdCall)]
1676
+ public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
1677
+ "@ -Name Mouse -Namespace Win32
1678
+ [Win32.Mouse]::mouse_event(0x0800, 0, 0, ${direction}, 0)`;
1679
+ await execAsync4(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
1680
+ } else if (process.platform === "darwin") {
1681
+ const dir = amount > 0 ? "u" : "d";
1682
+ await execAsync4(`cliclick -r ${dir}:${Math.abs(amount)}`);
1683
+ } else {
1684
+ const btn = amount > 0 ? "4" : "5";
1685
+ await execAsync4(`xdotool click --repeat ${Math.abs(amount)} ${btn}`);
1686
+ }
1687
+ return ok(`Scrolled ${amount > 0 ? "up" : "down"} by ${Math.abs(amount)}`);
1688
+ } catch (error) {
1689
+ return err(`Failed to scroll: ${error instanceof Error ? error.message : "Unknown error"}`);
1690
+ }
1691
+ }
1692
+ async function getMousePosition() {
1693
+ try {
1694
+ if (process.platform === "win32") {
1695
+ const { stdout } = await execAsync4(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; $p = [System.Windows.Forms.Cursor]::Position; Write-Output \\"$($p.X),$($p.Y)\\""`, { shell: "cmd.exe" });
1696
+ return ok(`Mouse position: ${stdout.trim()}`);
1697
+ } else if (process.platform === "darwin") {
1698
+ const { stdout } = await execAsync4(`cliclick p`);
1699
+ return ok(`Mouse position: ${stdout.trim()}`);
1700
+ } else {
1701
+ const { stdout } = await execAsync4(`xdotool getmouselocation --shell`);
1702
+ return ok(stdout);
1703
+ }
1704
+ } catch (error) {
1705
+ return err(`Failed to get mouse position: ${error instanceof Error ? error.message : "Unknown error"}`);
1706
+ }
1707
+ }
1421
1708
 
1422
1709
  // src/tools/index.ts
1423
1710
  function ok(output) {
@@ -1642,6 +1929,12 @@ ${result.error}
1642
1929
  }
1643
1930
  try {
1644
1931
  await ctx.sendChatAction("typing");
1932
+ const computerControlResult = await this.tryComputerControl(userText);
1933
+ if (computerControlResult) {
1934
+ await ctx.reply(computerControlResult);
1935
+ history.push({ role: "assistant", content: computerControlResult });
1936
+ return;
1937
+ }
1645
1938
  const isVisionRequest = /screen|see|look|what('?s| is) (on|visible)|show me|screenshot/i.test(userText);
1646
1939
  let response;
1647
1940
  if (isVisionRequest) {
@@ -1683,6 +1976,85 @@ ${result.error}
1683
1976
  }
1684
1977
  return this.allowedChatIds.has(chatId);
1685
1978
  }
1979
+ /**
1980
+ * Try to execute computer control commands directly
1981
+ * Returns response string if handled, null if not a computer command
1982
+ */
1983
+ async tryComputerControl(text) {
1984
+ const lower = text.toLowerCase();
1985
+ let match = lower.match(/minimize\s+(?:the\s+)?(.+)/i);
1986
+ if (match) {
1987
+ const result = await minimizeWindow(match[1].trim());
1988
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
1989
+ }
1990
+ match = lower.match(/maximize\s+(?:the\s+)?(.+)/i);
1991
+ if (match) {
1992
+ const result = await maximizeWindow(match[1].trim());
1993
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
1994
+ }
1995
+ match = lower.match(/close\s+(?:the\s+)?(.+)/i);
1996
+ if (match) {
1997
+ const result = await closeWindow(match[1].trim());
1998
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
1999
+ }
2000
+ match = lower.match(/restore\s+(?:the\s+)?(.+)/i);
2001
+ if (match) {
2002
+ const result = await restoreWindow(match[1].trim());
2003
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2004
+ }
2005
+ match = lower.match(/(?:focus|open|switch to)\s+(?:the\s+)?(.+)/i);
2006
+ if (match) {
2007
+ const result = await focusWindow(match[1].trim());
2008
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2009
+ }
2010
+ match = text.match(/type\s+["'](.+)["']/i);
2011
+ if (match) {
2012
+ const result = await typeText(match[1]);
2013
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2014
+ }
2015
+ match = lower.match(/press\s+(?:the\s+)?(\w+)/i);
2016
+ if (match) {
2017
+ const result = await pressKey(match[1]);
2018
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2019
+ }
2020
+ if (/^click$/i.test(lower) || /click\s+(?:the\s+)?mouse/i.test(lower)) {
2021
+ const result = await clickMouse("left");
2022
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2023
+ }
2024
+ if (/right\s*click/i.test(lower)) {
2025
+ const result = await clickMouse("right");
2026
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2027
+ }
2028
+ if (/double\s*click/i.test(lower)) {
2029
+ const result = await doubleClick();
2030
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2031
+ }
2032
+ match = lower.match(/move\s+(?:the\s+)?mouse\s+(?:to\s+)?(\d+)[,\s]+(\d+)/i);
2033
+ if (match) {
2034
+ const result = await moveMouse(parseInt(match[1]), parseInt(match[2]));
2035
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2036
+ }
2037
+ match = lower.match(/scroll\s+(up|down)(?:\s+(\d+))?/i);
2038
+ if (match) {
2039
+ const amount = match[1] === "up" ? parseInt(match[2]) || 3 : -(parseInt(match[2]) || 3);
2040
+ const result = await scrollMouse(amount);
2041
+ return result.success ? `\u2705 ${result.output}` : `\u274C ${result.error}`;
2042
+ }
2043
+ if (/list\s+(?:all\s+)?windows/i.test(lower) || /what\s+windows/i.test(lower)) {
2044
+ const result = await listWindows();
2045
+ return result.success ? `\u{1F4CB} Open Windows:
2046
+ ${result.output}` : `\u274C ${result.error}`;
2047
+ }
2048
+ if (/(?:active|current|focused)\s+window/i.test(lower) || /what\s+(?:window|app)/i.test(lower)) {
2049
+ const result = await getActiveWindow();
2050
+ return result.success ? `\u{1FA9F} Active: ${result.output}` : `\u274C ${result.error}`;
2051
+ }
2052
+ if (/mouse\s+position/i.test(lower) || /where.*mouse/i.test(lower)) {
2053
+ const result = await getMousePosition();
2054
+ return result.success ? `\u{1F5B1}\uFE0F ${result.output}` : `\u274C ${result.error}`;
2055
+ }
2056
+ return null;
2057
+ }
1686
2058
  /**
1687
2059
  * Send a message to a specific chat
1688
2060
  */
@@ -2311,17 +2683,33 @@ ${tasks.format(task)}`);
2311
2683
  setStatus("Ready");
2312
2684
  }
2313
2685
  }, [chat2, tasks]);
2686
+ const isComputerControlRequest = useCallback5((text) => {
2687
+ const lower = text.toLowerCase();
2688
+ const patterns = [
2689
+ /^(can you |please |)?(open|close|minimize|maximize|restore|focus|click|type|press|scroll|move|drag)/i,
2690
+ /^(can you |please |)?move (the |my |)mouse/i,
2691
+ /^(can you |please |)?(start|launch|run) [a-z]/i,
2692
+ /(open|close|minimize|maximize) (the |my |)?[a-z]/i,
2693
+ /click (on |the |)/i,
2694
+ /type ["'].+["']/i,
2695
+ /press (enter|escape|tab|ctrl|alt|shift|space|backspace|delete|f\d+)/i
2696
+ ];
2697
+ return patterns.some((p) => p.test(lower));
2698
+ }, []);
2314
2699
  const handleSubmit = useCallback5(async (value) => {
2315
2700
  if (!value.trim()) return;
2316
2701
  setInputValue("");
2317
2702
  if (value.startsWith("/")) {
2318
2703
  await handleCommand(value);
2704
+ } else if (isComputerControlRequest(value)) {
2705
+ chat2.addSystemMessage(`\u{1F916} Executing: ${value}`);
2706
+ await handleTaskCommand(value);
2319
2707
  } else {
2320
2708
  setStatus("Thinking...");
2321
2709
  await chat2.sendMessage(value);
2322
2710
  setStatus("Ready");
2323
2711
  }
2324
- }, [chat2, handleCommand]);
2712
+ }, [chat2, handleCommand, handleTaskCommand, isComputerControlRequest]);
2325
2713
  const handleProviderSelect = useCallback5((provider, model) => {
2326
2714
  chat2.addSystemMessage(`\u2705 Updated: ${provider} / ${model}`);
2327
2715
  }, [chat2]);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@projectservan8n/cnapse",
3
- "version": "0.6.0",
3
+ "version": "0.6.1",
4
4
  "description": "Autonomous PC intelligence - AI assistant for desktop automation",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -202,6 +202,21 @@ export function App() {
202
202
  }
203
203
  }, [chat, tasks]);
204
204
 
205
+ // Check if message looks like a computer control request
206
+ const isComputerControlRequest = useCallback((text: string): boolean => {
207
+ const lower = text.toLowerCase();
208
+ const patterns = [
209
+ /^(can you |please |)?(open|close|minimize|maximize|restore|focus|click|type|press|scroll|move|drag)/i,
210
+ /^(can you |please |)?move (the |my |)mouse/i,
211
+ /^(can you |please |)?(start|launch|run) [a-z]/i,
212
+ /(open|close|minimize|maximize) (the |my |)?[a-z]/i,
213
+ /click (on |the |)/i,
214
+ /type ["'].+["']/i,
215
+ /press (enter|escape|tab|ctrl|alt|shift|space|backspace|delete|f\d+)/i,
216
+ ];
217
+ return patterns.some(p => p.test(lower));
218
+ }, []);
219
+
205
220
  // Submit handler
206
221
  const handleSubmit = useCallback(async (value: string) => {
207
222
  if (!value.trim()) return;
@@ -209,12 +224,16 @@ export function App() {
209
224
 
210
225
  if (value.startsWith('/')) {
211
226
  await handleCommand(value);
227
+ } else if (isComputerControlRequest(value)) {
228
+ // Auto-route to task system for computer control
229
+ chat.addSystemMessage(`🤖 Executing: ${value}`);
230
+ await handleTaskCommand(value);
212
231
  } else {
213
232
  setStatus('Thinking...');
214
233
  await chat.sendMessage(value);
215
234
  setStatus('Ready');
216
235
  }
217
- }, [chat, handleCommand]);
236
+ }, [chat, handleCommand, handleTaskCommand, isComputerControlRequest]);
218
237
 
219
238
  // Provider selection callback
220
239
  const handleProviderSelect = useCallback((provider: string, model: string) => {
@@ -7,6 +7,7 @@ import { getConfig, getApiKey } from '../lib/config.js';
7
7
  import { describeScreen, captureScreenshot } from '../lib/vision.js';
8
8
  import { runCommand } from '../tools/shell.js';
9
9
  import { chat as chatWithAI, chatWithVision, Message } from '../lib/api.js';
10
+ import * as computer from '../tools/computer.js';
10
11
 
11
12
  export interface TelegramMessage {
12
13
  chatId: number;
@@ -248,6 +249,14 @@ export class TelegramBotService extends EventEmitter {
248
249
  // Send typing indicator
249
250
  await ctx.sendChatAction('typing');
250
251
 
252
+ // Check if this looks like a computer control request
253
+ const computerControlResult = await this.tryComputerControl(userText);
254
+ if (computerControlResult) {
255
+ await ctx.reply(computerControlResult);
256
+ history.push({ role: 'assistant', content: computerControlResult });
257
+ return;
258
+ }
259
+
251
260
  // Check if this looks like a screen/vision request
252
261
  const isVisionRequest = /screen|see|look|what('?s| is) (on|visible)|show me|screenshot/i.test(userText);
253
262
 
@@ -302,6 +311,117 @@ export class TelegramBotService extends EventEmitter {
302
311
  return this.allowedChatIds.has(chatId);
303
312
  }
304
313
 
314
+ /**
315
+ * Try to execute computer control commands directly
316
+ * Returns response string if handled, null if not a computer command
317
+ */
318
+ private async tryComputerControl(text: string): Promise<string | null> {
319
+ const lower = text.toLowerCase();
320
+
321
+ // Minimize window
322
+ let match = lower.match(/minimize\s+(?:the\s+)?(.+)/i);
323
+ if (match) {
324
+ const result = await computer.minimizeWindow(match[1].trim());
325
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
326
+ }
327
+
328
+ // Maximize window
329
+ match = lower.match(/maximize\s+(?:the\s+)?(.+)/i);
330
+ if (match) {
331
+ const result = await computer.maximizeWindow(match[1].trim());
332
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
333
+ }
334
+
335
+ // Close window
336
+ match = lower.match(/close\s+(?:the\s+)?(.+)/i);
337
+ if (match) {
338
+ const result = await computer.closeWindow(match[1].trim());
339
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
340
+ }
341
+
342
+ // Restore window
343
+ match = lower.match(/restore\s+(?:the\s+)?(.+)/i);
344
+ if (match) {
345
+ const result = await computer.restoreWindow(match[1].trim());
346
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
347
+ }
348
+
349
+ // Focus/open window
350
+ match = lower.match(/(?:focus|open|switch to)\s+(?:the\s+)?(.+)/i);
351
+ if (match) {
352
+ const result = await computer.focusWindow(match[1].trim());
353
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
354
+ }
355
+
356
+ // Type text
357
+ match = text.match(/type\s+["'](.+)["']/i);
358
+ if (match) {
359
+ const result = await computer.typeText(match[1]);
360
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
361
+ }
362
+
363
+ // Press key
364
+ match = lower.match(/press\s+(?:the\s+)?(\w+)/i);
365
+ if (match) {
366
+ const result = await computer.pressKey(match[1]);
367
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
368
+ }
369
+
370
+ // Click
371
+ if (/^click$/i.test(lower) || /click\s+(?:the\s+)?mouse/i.test(lower)) {
372
+ const result = await computer.clickMouse('left');
373
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
374
+ }
375
+
376
+ // Right click
377
+ if (/right\s*click/i.test(lower)) {
378
+ const result = await computer.clickMouse('right');
379
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
380
+ }
381
+
382
+ // Double click
383
+ if (/double\s*click/i.test(lower)) {
384
+ const result = await computer.doubleClick();
385
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
386
+ }
387
+
388
+ // Move mouse to coordinates
389
+ match = lower.match(/move\s+(?:the\s+)?mouse\s+(?:to\s+)?(\d+)[,\s]+(\d+)/i);
390
+ if (match) {
391
+ const result = await computer.moveMouse(parseInt(match[1]), parseInt(match[2]));
392
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
393
+ }
394
+
395
+ // Scroll
396
+ match = lower.match(/scroll\s+(up|down)(?:\s+(\d+))?/i);
397
+ if (match) {
398
+ const amount = match[1] === 'up' ? (parseInt(match[2]) || 3) : -(parseInt(match[2]) || 3);
399
+ const result = await computer.scrollMouse(amount);
400
+ return result.success ? `✅ ${result.output}` : `❌ ${result.error}`;
401
+ }
402
+
403
+ // List windows
404
+ if (/list\s+(?:all\s+)?windows/i.test(lower) || /what\s+windows/i.test(lower)) {
405
+ const result = await computer.listWindows();
406
+ return result.success ? `📋 Open Windows:\n${result.output}` : `❌ ${result.error}`;
407
+ }
408
+
409
+ // Get active window
410
+ if (/(?:active|current|focused)\s+window/i.test(lower) || /what\s+(?:window|app)/i.test(lower)) {
411
+ const result = await computer.getActiveWindow();
412
+ return result.success ? `🪟 Active: ${result.output}` : `❌ ${result.error}`;
413
+ }
414
+
415
+ // Mouse position
416
+ if (/mouse\s+position/i.test(lower) || /where.*mouse/i.test(lower)) {
417
+ const result = await computer.getMousePosition();
418
+ return result.success ? `🖱️ ${result.output}` : `❌ ${result.error}`;
419
+ }
420
+
421
+ // Not a computer control command
422
+ return null;
423
+ }
424
+
305
425
  /**
306
426
  * Send a message to a specific chat
307
427
  */