@mseep/clawdcursor 1.5.5

package/dist/tools/shortcuts.js

@@ -0,0 +1,205 @@
+"use strict";
+/**
+ * Shortcut tools — expose the keyboard shortcuts database to MCP clients.
+ *
+ * Two tools:
+ *   shortcuts_list    — query available shortcuts by category/context
+ *   shortcuts_execute — run a shortcut by intent (fuzzy-matched)
+ *
+ * This bridges the gap between the pipeline router (`src/core/router/router.ts`)
+ * — which knows all shortcuts — and external agents calling MCP tools
+ * that previously had to independently know keyboard combos.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getShortcutTools = getShortcutTools;
+const os = __importStar(require("os"));
+const shortcuts_1 = require("../shortcuts");
+const VALID_CATEGORIES = [
+    'navigation', 'browser', 'editing', 'social', 'window', 'file', 'view', 'quick',
+];
+/**
+ * Build a compact shortcut entry for the list response.
+ * Includes the resolved key combo for the current platform.
+ */
+function formatShortcut(s, platform) {
+    return {
+        id: s.id,
+        category: s.category,
+        description: s.description,
+        intent: s.canonicalIntent,
+        key: (0, shortcuts_1.resolveShortcutKey)(s, platform),
+        context: s.contextHints?.length ? s.contextHints : undefined,
+    };
+}
+function getShortcutTools() {
+    return [
+        {
+            name: 'shortcuts_list',
+            description: 'List available keyboard shortcuts. Filter by category (navigation, browser, editing, social, window, file, view, quick) and/or context (e.g. "reddit", "outlook"). Returns shortcut names, descriptions, and key combos for the current platform. Use this BEFORE reaching for mouse_scroll or mouse_click — there is often a faster keyboard shortcut.',
+            parameters: {
+                category: {
+                    type: 'string',
+                    description: `Filter by category: ${VALID_CATEGORIES.join(', ')}. Omit to list all.`,
+                    required: false,
+                    enum: VALID_CATEGORIES,
+                },
+                context: {
+                    type: 'string',
+                    description: 'Filter by app context (e.g. "reddit", "outlook", "x.com"). Shows context-specific shortcuts that match.',
+                    required: false,
+                },
+            },
+            category: 'keyboard',
+            compactGroup: 'system',
+            safetyTier: 0,
+            handler: async ({ category, context }) => {
+                const platform = os.platform();
+                let filtered = shortcuts_1.SHORTCUTS;
+                // Filter by category
+                if (category) {
+                    filtered = filtered.filter(s => s.category === category);
+                }
+                // Filter by context — include shortcuts with no context hints (universal)
+                // plus those whose contextHints match the given context
+                if (context) {
+                    const ctx = context.toLowerCase();
+                    filtered = filtered.filter(s => {
+                        if (!s.contextHints?.length)
+                            return true; // universal shortcut
+                        return s.contextHints.some(h => h.toLowerCase().includes(ctx) || ctx.includes(h.toLowerCase()));
+                    });
+                }
+                else {
+                    // When no context given, exclude context-specific shortcuts
+                    // (they'd be confusing without context — e.g. reddit "a" for upvote)
+                    filtered = filtered.filter(s => !s.contextHints?.length);
+                }
+                // Drop shortcuts that have NO key on this platform (e.g. macOS-only
+                // entries like Hide-Others/App-Exposé resolve to "" on Windows/Linux),
+                // and dedupe by id so a stray duplicate definition can't surface the
+                // same shortcut twice (gauntlet F6).
+                const seen = new Set();
+                const results = filtered
+                    .map(s => formatShortcut(s, platform))
+                    .filter(r => {
+                    if (!r.key || r.key.trim() === '')
+                        return false;
+                    if (seen.has(r.id))
+                        return false;
+                    seen.add(r.id);
+                    return true;
+                });
+                if (results.length === 0) {
+                    return {
+                        text: `No shortcuts found${category ? ` in category "${category}"` : ''}${context ? ` for context "${context}"` : ''}. Available categories: ${VALID_CATEGORIES.join(', ')}`,
+                    };
+                }
+                return {
+                    text: JSON.stringify({
+                        platform,
+                        count: results.length,
+                        shortcuts: results,
+                        hint: 'Run a shortcut with the system compound: { action: "shortcuts_run", intent: "<intent>" }. Or press the key combo directly via computer.key.',
+                    }, null, 2),
+                };
+            },
+        },
+        {
+            name: 'shortcuts_execute',
+            description: 'Execute a keyboard shortcut by describing what you want to do (e.g. "scroll down", "close tab", "upvote"). Uses fuzzy matching against the shortcuts database. Provide context (active app name) for app-specific shortcuts like Reddit or Outlook.',
+            parameters: {
+                intent: {
+                    type: 'string',
+                    description: 'What you want to do (e.g. "scroll down", "new tab", "copy", "upvote", "reply to email")',
+                    required: true,
+                },
+                context: {
+                    type: 'string',
+                    description: 'Active app context for app-specific shortcuts (e.g. "reddit", "outlook"). Auto-detected from active window if omitted.',
+                    required: false,
+                },
+            },
+            category: 'keyboard',
+            compactGroup: 'system',
+            safetyTier: 1,
+            handler: async ({ intent, context }, ctx) => {
+                await ctx.ensureInitialized();
+                // Build context hint — use provided context or detect from active window
+                let contextHint = context ?? '';
+                if (!contextHint) {
+                    try {
+                        const win = await ctx.a11y.getActiveWindow();
+                        if (win) {
+                            contextHint = `${win.processName ?? ''} ${win.title ?? ''}`;
+                        }
+                    }
+                    catch { /* non-fatal — proceed without context */ }
+                }
+                const match = (0, shortcuts_1.findShortcut)(intent, os.platform(), {
+                    contextHint,
+                    enableFuzzy: true,
+                });
+                if (!match) {
+                    // Return helpful error with similar shortcuts
+                    const suggestions = shortcuts_1.SHORTCUTS
+                        .filter(s => !s.contextHints?.length || (contextHint && s.contextHints.some(h => contextHint.toLowerCase().includes(h.toLowerCase()))))
+                        .slice(0, 10)
+                        .map(s => `  • "${s.canonicalIntent}" → ${(0, shortcuts_1.resolveShortcutKey)(s, os.platform())} (${s.description})`);
+                    return {
+                        text: `No shortcut matched intent "${intent}". Try one of these:\n${suggestions.join('\n')}\n\nOr use key_press directly with a specific key combo.`,
+                        isError: true,
+                    };
+                }
+                // Execute the matched shortcut
+                const active = await ctx.a11y.getActiveWindow().catch(() => null);
+                const activeInfo = active ? `[${active.processName}] "${active.title}"` : '(unknown)';
+                await ctx.desktop.keyPress(match.combo);
+                ctx.a11y.invalidateCache();
+                return {
+                    text: JSON.stringify({
+                        executed: match.combo,
+                        intent: match.canonicalIntent,
+                        matched: match.matchedIntent,
+                        matchType: match.matchType,
+                        description: match.shortcut.description,
+                        window: activeInfo,
+                    }),
+                };
+            },
+        },
+    ];
+}
+//# sourceMappingURL=shortcuts.js.map

package/dist/tools/shortcuts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shortcuts.js","sourceRoot":"","sources":["../../src/tools/shortcuts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BH,4CAmJC;AAhLD,uCAAyB;AACzB,4CAMsB;AAGtB,MAAM,gBAAgB,GAAuB;IAC3C,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;CAChF,CAAC;AAEF;;;GAGG;AACH,SAAS,cAAc,CAAC,CAAqB,EAAE,QAAyB;IACtE,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,MAAM,EAAE,CAAC,CAAC,eAAe;QACzB,GAAG,EAAE,IAAA,8BAAkB,EAAC,CAAC,EAAE,QAAQ,CAAC;QACpC,OAAO,EAAE,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;KAC7D,CAAC;AACJ,CAAC;AAED,SAAgB,gBAAgB;IAC9B,OAAO;QACL;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EACT,yVAAyV;YAC3V,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uBAAuB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB;oBACpF,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,gBAAuC;iBAC9C;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,yGAAyG;oBACtH,QAAQ,EAAE,KAAK;iBAChB;aACF;YACD,QAAQ,EAAE,UAAU;YACpB,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE;gBACvC,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAC/B,IAAI,QAAQ,GAAG,qBAAS,CAAC;gBAEzB,qBAAqB;gBACrB,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;gBAC3D,CAAC;gBAED,0EAA0E;gBAC1E,wDAAwD;gBACxD,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;oBAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;wBAC7B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,MAAM;4BAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;wBAC/D,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;oBAClG,CAAC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,4DAA4D;oBAC5D,qEAAqE;oBACrE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC3D,CAAC;gBAED,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,qCAAqC;gBACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;gBAC/B,MAAM,OAAO,GAAG,QAAQ;qBACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,QAAQ,CAAgC,CAAC;qBACpE,MAAM,CAAC,CAAC,CAAC,EAAE;oBACV,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;wBAAE,OAAO,KAAK,CAAC;oBAChD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;wBAAE,OAAO,KAAK,CAAC;oBACjC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACf,OAAO,IAAI,CAAC;gBACd,CAAC,CAAC,CAAC;gBAEL,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO;wBACL,IAAI,EAAE,qBAAqB,QAAQ,CAAC,CAAC,CAAC,iBAAiB,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,iBAAiB,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,2BAA2B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBAC7K,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,QAAQ;wBACR,KAAK,EAAE,OAAO,CAAC,MAAM;wBACrB,SAAS,EAAE,OAAO;wBAClB,IAAI,EAAE,6IAA6I;qBACpJ,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;YACJ,CAAC;SACF;QAED;YACE,IAAI,EAAE,mBAAmB;YACzB,WAAW,EACT,qPAAqP;YACvP,UAAU,EAAE;gBACV,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,yFAAyF;oBACtG,QAAQ,EAAE,IAAI;iBACf;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,wHAAwH;oBACrI,QAAQ,EAAE,KAAK;iBAChB;aACF;YACD,QAAQ,EAAE,UAAU;YACpB,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE;gBAC1C,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAE9B,yEAAyE;gBACzE,IAAI,WAAW,GAAG,OAAO,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;wBAC7C,IAAI,GAAG,EAAE,CAAC;4BACR,WAAW,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;wBAC9D,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC,CAAC,yCAAyC,CAAC,CAAC;gBACvD,CAAC;gBAED,MAAM,KAAK,GAAG,IAAA,wBAAY,EAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE;oBAChD,WAAW;oBACX,WAAW,EAAE,IAAI;iBAClB,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,8CAA8C;oBAC9C,MAAM,WAAW,GAAG,qBAAS;yBAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;yBACtI,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;yBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,eAAe,OAAO,IAAA,8BAAkB,EAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;oBAEvG,OAAO;wBACL,IAAI,EAAE,+BAA+B,MAAM,yBAAyB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,0DAA0D;wBACpJ,OAAO,EAAE,IAAI;qBACd,CAAC;gBACJ,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;gBAClE,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,MAAM,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;gBAEtF,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAE3B,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,QAAQ,EAAE,KAAK,CAAC,KAAK;wBACrB,MAAM,EAAE,KAAK,CAAC,eAAe;wBAC7B,OAAO,EAAE,KAAK,CAAC,aAAa;wBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;wBACvC,MAAM,EAAE,UAAU;qBACnB,CAAC;iBACH,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC"}

package/dist/tools/smart.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Smart tools — high-level name-based interaction for blind agents.
+ *
+ * These tools let MCP clients interact with the desktop WITHOUT needing
+ * screenshots or coordinate math.
+ *
+ * Perception order: OCR first (primary), a11y tree in parallel (supplement).
+ * If combined OCR+a11y can't handle it → CDP fallback → fail.
+ * Vision is never called from here — that's the caller's last resort.
+ *
+ * Key design: NO coordinate conversion needed by the caller.
+ * OCR coords and a11y coords both match nut-js mouseClick coords directly.
+ */
+import type { ToolDefinition } from './types';
+export declare function getSmartTools(): ToolDefinition[];