@kortix/agent-tunnel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +37 -0
- package/src/agent/agent.ts +331 -0
- package/src/agent/capabilities/desktop/atspi-helper.ts +345 -0
- package/src/agent/capabilities/desktop/csharp-helper.ts +914 -0
- package/src/agent/capabilities/desktop/linux-driver.ts +368 -0
- package/src/agent/capabilities/desktop/macos-driver.ts +601 -0
- package/src/agent/capabilities/desktop/swift-helper.ts +736 -0
- package/src/agent/capabilities/desktop/types.ts +201 -0
- package/src/agent/capabilities/desktop/windows-driver.ts +220 -0
- package/src/agent/capabilities/desktop.ts +196 -0
- package/src/agent/capabilities/filesystem.ts +133 -0
- package/src/agent/capabilities/index.ts +42 -0
- package/src/agent/capabilities/shell.ts +96 -0
- package/src/agent/cli.ts +222 -0
- package/src/agent/config.ts +54 -0
- package/src/agent/index.ts +11 -0
- package/src/agent/security/command-validator.ts +61 -0
- package/src/agent/security/path-validator.ts +55 -0
- package/src/agent/security/permission-guard.ts +66 -0
- package/src/client/index.ts +4 -0
- package/src/client/tools.ts +603 -0
- package/src/client/tunnel-client.ts +270 -0
- package/src/index.ts +62 -0
- package/src/server/heartbeat.ts +84 -0
- package/src/server/index.ts +7 -0
- package/src/server/relay.ts +266 -0
- package/src/server/routes.ts +61 -0
- package/src/server/server.ts +114 -0
- package/src/server/ws-handler.ts +54 -0
- package/src/shared/crypto.ts +58 -0
- package/src/shared/index.ts +33 -0
- package/src/shared/types.ts +164 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Permission Guard — local-side permission enforcement (defense in depth).
|
|
3
|
+
*
|
|
4
|
+
* Even though the server validates permissions before relaying RPCs,
|
|
5
|
+
* the local agent also checks permissions as a second layer of defense.
|
|
6
|
+
* This prevents a compromised server from bypassing permission controls.
|
|
7
|
+
*
|
|
8
|
+
* After the initial permission sync, unknown permissionIds are denied.
|
|
9
|
+
* Before sync, unknown IDs are also denied (fail-closed).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export interface LocalPermission {
|
|
13
|
+
permissionId: string;
|
|
14
|
+
capability: string;
|
|
15
|
+
scope: Record<string, unknown>;
|
|
16
|
+
expiresAt?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class PermissionGuard {
|
|
20
|
+
private permissions = new Map<string, LocalPermission>();
|
|
21
|
+
private hasSynced = false;
|
|
22
|
+
|
|
23
|
+
/** Bulk-load permissions from server sync notification. */
|
|
24
|
+
syncPermissions(permissions: LocalPermission[]): void {
|
|
25
|
+
this.permissions.clear();
|
|
26
|
+
for (const perm of permissions) {
|
|
27
|
+
this.permissions.set(perm.permissionId, perm);
|
|
28
|
+
}
|
|
29
|
+
this.hasSynced = true;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
addPermission(permission: LocalPermission): void {
|
|
33
|
+
this.permissions.set(permission.permissionId, permission);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
revokePermission(permissionId: string): void {
|
|
37
|
+
this.permissions.delete(permissionId);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
checkPermission(permissionId: string | undefined): boolean {
|
|
41
|
+
if (!permissionId) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const perm = this.permissions.get(permissionId);
|
|
46
|
+
if (!perm) {
|
|
47
|
+
// After sync, unknown permission = deny (fail-closed).
|
|
48
|
+
// Before sync, also deny — we have no basis to allow.
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if (perm.expiresAt) {
|
|
53
|
+
if (new Date(perm.expiresAt) < new Date()) {
|
|
54
|
+
this.permissions.delete(permissionId);
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
clear(): void {
|
|
63
|
+
this.permissions.clear();
|
|
64
|
+
this.hasSynced = false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,603 @@
|
|
|
1
|
+
import { writeFileSync, mkdirSync } from 'fs';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { tmpdir } from 'os';
|
|
4
|
+
import { randomBytes } from 'crypto';
|
|
5
|
+
import type { TunnelClient, AXElement } from './tunnel-client';
|
|
6
|
+
|
|
7
|
+
export interface TunnelToolParameter {
|
|
8
|
+
type: string;
|
|
9
|
+
description: string;
|
|
10
|
+
required?: boolean;
|
|
11
|
+
items?: { type: string };
|
|
12
|
+
enum?: string[];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface TunnelToolDefinition {
|
|
16
|
+
name: string;
|
|
17
|
+
description: string;
|
|
18
|
+
parameters: Record<string, TunnelToolParameter>;
|
|
19
|
+
execute: (args: Record<string, unknown>) => Promise<string>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function saveImage(base64: string, format: string): string {
|
|
23
|
+
const ext = format === 'jpeg' || format === 'jpg' ? 'jpg' : 'png';
|
|
24
|
+
const dir = join(tmpdir(), 'tunnel-screenshots');
|
|
25
|
+
mkdirSync(dir, { recursive: true });
|
|
26
|
+
const path = join(dir, `screenshot-${randomBytes(4).toString('hex')}.${ext}`);
|
|
27
|
+
writeFileSync(path, Buffer.from(base64, 'base64'));
|
|
28
|
+
return path;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function formatAXTree(el: AXElement, indent: number = 0): string {
|
|
32
|
+
const pad = ' '.repeat(indent);
|
|
33
|
+
const parts: string[] = [];
|
|
34
|
+
|
|
35
|
+
const label = el.title || el.value || el.description || '(unnamed)';
|
|
36
|
+
const flags: string[] = [];
|
|
37
|
+
if (!el.enabled) flags.push('disabled');
|
|
38
|
+
if (el.focused) flags.push('focused');
|
|
39
|
+
if (el.actions.length > 0) flags.push(`actions: ${el.actions.join(',')}`);
|
|
40
|
+
const flagStr = flags.length > 0 ? ` [${flags.join(', ')}]` : '';
|
|
41
|
+
|
|
42
|
+
parts.push(`${pad}[${el.role}] ${label} (id: ${el.id})${flagStr}`);
|
|
43
|
+
|
|
44
|
+
for (const child of el.children) {
|
|
45
|
+
parts.push(formatAXTree(child, indent + 1));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return parts.join('\n');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const tunnelIdParam: TunnelToolParameter = {
|
|
52
|
+
type: 'string',
|
|
53
|
+
description: 'Tunnel connection ID (auto-discovered if omitted)',
|
|
54
|
+
required: false,
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export function createTunnelTools(client: TunnelClient): TunnelToolDefinition[] {
|
|
58
|
+
return [
|
|
59
|
+
{
|
|
60
|
+
name: 'tunnel_status',
|
|
61
|
+
description: `Check the status of all Agent Tunnel connections to the user's local machine. Lists every registered tunnel with its live/offline status, capabilities, and machine info.`,
|
|
62
|
+
parameters: {},
|
|
63
|
+
async execute() {
|
|
64
|
+
const connections = (await client.getConnections()) as Array<Record<string, unknown>>;
|
|
65
|
+
|
|
66
|
+
if (connections.length === 0) {
|
|
67
|
+
return 'No tunnel connections found. The user needs to set up Agent Tunnel first:\n1. Create a tunnel connection\n2. Run `npx @kortix/agent-tunnel connect` on their local machine';
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const sections: string[] = [];
|
|
71
|
+
let hasOnline = false;
|
|
72
|
+
|
|
73
|
+
for (const data of connections) {
|
|
74
|
+
const status = data.isLive ? 'ONLINE' : 'OFFLINE';
|
|
75
|
+
if (data.isLive) hasOnline = true;
|
|
76
|
+
const capabilities = (data.capabilities as string[]) || [];
|
|
77
|
+
const machineInfo = (data.machineInfo as Record<string, unknown>) || {};
|
|
78
|
+
|
|
79
|
+
const lines = [
|
|
80
|
+
`=== Tunnel: ${data.name || 'Unnamed'} — ${status} ===`,
|
|
81
|
+
`ID: ${data.tunnelId}`,
|
|
82
|
+
`Capabilities: ${capabilities.length > 0 ? capabilities.join(', ') : '(none registered)'}`,
|
|
83
|
+
];
|
|
84
|
+
|
|
85
|
+
if (Object.keys(machineInfo).length > 0) {
|
|
86
|
+
lines.push(`Machine: ${machineInfo.hostname || 'unknown'} (${machineInfo.platform || '?'} ${machineInfo.arch || '?'})`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
sections.push(lines.join('\n'));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (!hasOnline) {
|
|
93
|
+
sections.push('\nNo tunnel is currently online. Ask the user to run `npx @kortix/agent-tunnel connect` on their local machine.');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return sections.join('\n\n');
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
name: 'tunnel_fs_read',
|
|
101
|
+
description: `Read a file from the user's local machine via Agent Tunnel. Requires filesystem permission.`,
|
|
102
|
+
parameters: {
|
|
103
|
+
tunnel_id: tunnelIdParam,
|
|
104
|
+
path: { type: 'string', description: 'Absolute path to the file on the user\'s local machine', required: true },
|
|
105
|
+
encoding: { type: 'string', description: 'File encoding (default: utf-8)', required: false },
|
|
106
|
+
},
|
|
107
|
+
async execute(args) {
|
|
108
|
+
const result = await client.rpcWithPermissionFlow('fs.read', {
|
|
109
|
+
path: args.path,
|
|
110
|
+
encoding: (args.encoding as string) || 'utf-8',
|
|
111
|
+
});
|
|
112
|
+
if (typeof result === 'string') return result;
|
|
113
|
+
const data = result as Record<string, unknown>;
|
|
114
|
+
return `=== File: ${args.path} (${data.size} bytes) ===\n${data.content}`;
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
name: 'tunnel_fs_write',
|
|
119
|
+
description: `Write a file to the user's local machine via Agent Tunnel. Creates parent directories if needed. Requires filesystem write permission.`,
|
|
120
|
+
parameters: {
|
|
121
|
+
tunnel_id: tunnelIdParam,
|
|
122
|
+
path: { type: 'string', description: 'Absolute path for the file on the user\'s local machine', required: true },
|
|
123
|
+
content: { type: 'string', description: 'File content to write', required: true },
|
|
124
|
+
encoding: { type: 'string', description: 'File encoding (default: utf-8)', required: false },
|
|
125
|
+
},
|
|
126
|
+
async execute(args) {
|
|
127
|
+
const result = await client.rpcWithPermissionFlow('fs.write', {
|
|
128
|
+
path: args.path,
|
|
129
|
+
content: args.content,
|
|
130
|
+
encoding: (args.encoding as string) || 'utf-8',
|
|
131
|
+
});
|
|
132
|
+
if (typeof result === 'string') return result;
|
|
133
|
+
const data = result as Record<string, unknown>;
|
|
134
|
+
return `File written: ${data.path} (${data.size} bytes)`;
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
name: 'tunnel_fs_list',
|
|
139
|
+
description: `List directory contents on the user's local machine via Agent Tunnel. Requires filesystem permission.`,
|
|
140
|
+
parameters: {
|
|
141
|
+
tunnel_id: tunnelIdParam,
|
|
142
|
+
path: { type: 'string', description: 'Absolute path to the directory on the user\'s local machine', required: true },
|
|
143
|
+
recursive: { type: 'boolean', description: 'Include subdirectory contents (default: false)', required: false },
|
|
144
|
+
},
|
|
145
|
+
async execute(args) {
|
|
146
|
+
const result = await client.rpcWithPermissionFlow('fs.list', {
|
|
147
|
+
path: args.path,
|
|
148
|
+
recursive: args.recursive || false,
|
|
149
|
+
});
|
|
150
|
+
if (typeof result === 'string') return result;
|
|
151
|
+
|
|
152
|
+
const data = result as { entries: Array<{ name: string; path: string; isDirectory: boolean; isFile: boolean }>; count: number };
|
|
153
|
+
if (data.entries.length === 0) return `Directory is empty: ${args.path}`;
|
|
154
|
+
|
|
155
|
+
const lines = [`=== Directory: ${args.path} (${data.count} entries) ===`];
|
|
156
|
+
for (const entry of data.entries) {
|
|
157
|
+
const type = entry.isDirectory ? '[DIR]' : '[FILE]';
|
|
158
|
+
lines.push(` ${type} ${entry.name}`);
|
|
159
|
+
}
|
|
160
|
+
return lines.join('\n');
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
name: 'tunnel_shell_exec',
|
|
165
|
+
description: `Execute a command on the user's local machine via Agent Tunnel. Commands are executed without shell interpolation (array args) for security. Requires shell permission.`,
|
|
166
|
+
parameters: {
|
|
167
|
+
tunnel_id: tunnelIdParam,
|
|
168
|
+
command: { type: 'string', description: "Command executable name (e.g., 'ls', 'git', 'python')", required: true },
|
|
169
|
+
args: { type: 'array', description: 'Command arguments as separate strings (no shell interpolation)', required: false, items: { type: 'string' } },
|
|
170
|
+
cwd: { type: 'string', description: 'Working directory for the command', required: false },
|
|
171
|
+
timeout: { type: 'number', description: 'Timeout in milliseconds (default: 30000, max: 120000)', required: false },
|
|
172
|
+
},
|
|
173
|
+
async execute(args) {
|
|
174
|
+
const result = await client.rpcWithPermissionFlow('shell.exec', {
|
|
175
|
+
command: args.command,
|
|
176
|
+
args: (args.args as string[]) || [],
|
|
177
|
+
cwd: args.cwd,
|
|
178
|
+
timeout: args.timeout,
|
|
179
|
+
});
|
|
180
|
+
if (typeof result === 'string') return result;
|
|
181
|
+
|
|
182
|
+
const data = result as {
|
|
183
|
+
exitCode: number | null;
|
|
184
|
+
signal: string | null;
|
|
185
|
+
stdout: string;
|
|
186
|
+
stderr: string;
|
|
187
|
+
stdoutTruncated: boolean;
|
|
188
|
+
stderrTruncated: boolean;
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
const lines = [`=== Command: ${args.command} ${((args.args as string[]) || []).join(' ')} ===`];
|
|
192
|
+
lines.push(`Exit code: ${data.exitCode ?? 'N/A'}${data.signal ? ` (signal: ${data.signal})` : ''}`);
|
|
193
|
+
if (data.stdout) {
|
|
194
|
+
lines.push(`\n--- stdout${data.stdoutTruncated ? ' (truncated)' : ''} ---`);
|
|
195
|
+
lines.push(data.stdout);
|
|
196
|
+
}
|
|
197
|
+
if (data.stderr) {
|
|
198
|
+
lines.push(`\n--- stderr${data.stderrTruncated ? ' (truncated)' : ''} ---`);
|
|
199
|
+
lines.push(data.stderr);
|
|
200
|
+
}
|
|
201
|
+
return lines.join('\n');
|
|
202
|
+
},
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
name: 'tunnel_screenshot',
|
|
206
|
+
description: `Take a screenshot of the user's screen via Agent Tunnel. Saves the image to a temp file and returns the path.`,
|
|
207
|
+
parameters: {
|
|
208
|
+
tunnel_id: tunnelIdParam,
|
|
209
|
+
x: { type: 'number', description: 'Region X coordinate', required: false },
|
|
210
|
+
y: { type: 'number', description: 'Region Y coordinate', required: false },
|
|
211
|
+
width: { type: 'number', description: 'Region width', required: false },
|
|
212
|
+
height: { type: 'number', description: 'Region height', required: false },
|
|
213
|
+
windowId: { type: 'number', description: 'Capture a specific window by ID', required: false },
|
|
214
|
+
},
|
|
215
|
+
async execute(args) {
|
|
216
|
+
const params: Record<string, unknown> = {};
|
|
217
|
+
if (args.x !== undefined && args.y !== undefined && args.width !== undefined && args.height !== undefined) {
|
|
218
|
+
params.region = { x: args.x, y: args.y, width: args.width, height: args.height };
|
|
219
|
+
}
|
|
220
|
+
if (args.windowId !== undefined) params.windowId = args.windowId;
|
|
221
|
+
|
|
222
|
+
const result = await client.rpcWithPermissionFlow('desktop.screenshot', params);
|
|
223
|
+
if (typeof result === 'string') return result;
|
|
224
|
+
|
|
225
|
+
const data = result as { image: string; width: number; height: number; format?: string };
|
|
226
|
+
const format = data.format || 'png';
|
|
227
|
+
const sizeKB = Math.round(data.image.length * 0.75 / 1024);
|
|
228
|
+
const path = saveImage(data.image, format);
|
|
229
|
+
return `Screenshot saved: ${path}\nDimensions: ${data.width}x${data.height} ${format.toUpperCase()} (${sizeKB}KB)\n\nUse the Read tool to view this image.`;
|
|
230
|
+
},
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
name: 'tunnel_click',
|
|
234
|
+
description: `Click at a specific screen coordinate on the user's machine via Agent Tunnel.`,
|
|
235
|
+
parameters: {
|
|
236
|
+
tunnel_id: tunnelIdParam,
|
|
237
|
+
x: { type: 'number', description: 'X coordinate to click', required: true },
|
|
238
|
+
y: { type: 'number', description: 'Y coordinate to click', required: true },
|
|
239
|
+
button: { type: 'string', description: 'Mouse button (default: left)', required: false, enum: ['left', 'right', 'middle'] },
|
|
240
|
+
clicks: { type: 'number', description: 'Number of clicks (default: 1, use 2 for double-click)', required: false },
|
|
241
|
+
modifiers: { type: 'array', description: 'Modifier keys held during click: cmd, shift, alt, ctrl', required: false, items: { type: 'string' } },
|
|
242
|
+
},
|
|
243
|
+
async execute(args) {
|
|
244
|
+
const result = await client.rpcWithPermissionFlow('desktop.mouse.click', {
|
|
245
|
+
x: args.x, y: args.y, button: args.button, clicks: args.clicks, modifiers: args.modifiers,
|
|
246
|
+
});
|
|
247
|
+
if (typeof result === 'string') return result;
|
|
248
|
+
return `Clicked at (${args.x}, ${args.y}) [${args.button || 'left'}]${args.clicks && (args.clicks as number) > 1 ? ` x${args.clicks}` : ''}`;
|
|
249
|
+
},
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
name: 'tunnel_mouse_move',
|
|
253
|
+
description: `Move the mouse cursor to a specific position on the user's screen via Agent Tunnel.`,
|
|
254
|
+
parameters: {
|
|
255
|
+
tunnel_id: tunnelIdParam,
|
|
256
|
+
x: { type: 'number', description: 'Target X coordinate', required: true },
|
|
257
|
+
y: { type: 'number', description: 'Target Y coordinate', required: true },
|
|
258
|
+
},
|
|
259
|
+
async execute(args) {
|
|
260
|
+
const result = await client.rpcWithPermissionFlow('desktop.mouse.move', { x: args.x, y: args.y });
|
|
261
|
+
if (typeof result === 'string') return result;
|
|
262
|
+
return `Mouse moved to (${args.x}, ${args.y})`;
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
name: 'tunnel_mouse_drag',
|
|
267
|
+
description: `Drag from one point to another on the user's screen via Agent Tunnel.`,
|
|
268
|
+
parameters: {
|
|
269
|
+
tunnel_id: tunnelIdParam,
|
|
270
|
+
fromX: { type: 'number', description: 'Start X coordinate', required: true },
|
|
271
|
+
fromY: { type: 'number', description: 'Start Y coordinate', required: true },
|
|
272
|
+
toX: { type: 'number', description: 'End X coordinate', required: true },
|
|
273
|
+
toY: { type: 'number', description: 'End Y coordinate', required: true },
|
|
274
|
+
button: { type: 'string', description: 'Mouse button (default: left)', required: false, enum: ['left', 'right'] },
|
|
275
|
+
},
|
|
276
|
+
async execute(args) {
|
|
277
|
+
const result = await client.rpcWithPermissionFlow('desktop.mouse.drag', {
|
|
278
|
+
fromX: args.fromX, fromY: args.fromY, toX: args.toX, toY: args.toY, button: args.button,
|
|
279
|
+
});
|
|
280
|
+
if (typeof result === 'string') return result;
|
|
281
|
+
return `Dragged from (${args.fromX}, ${args.fromY}) to (${args.toX}, ${args.toY})`;
|
|
282
|
+
},
|
|
283
|
+
},
|
|
284
|
+
{
|
|
285
|
+
name: 'tunnel_mouse_scroll',
|
|
286
|
+
description: `Scroll the mouse wheel at a specific position on the user's screen via Agent Tunnel.`,
|
|
287
|
+
parameters: {
|
|
288
|
+
tunnel_id: tunnelIdParam,
|
|
289
|
+
x: { type: 'number', description: 'X coordinate to scroll at', required: true },
|
|
290
|
+
y: { type: 'number', description: 'Y coordinate to scroll at', required: true },
|
|
291
|
+
deltaX: { type: 'number', description: 'Horizontal scroll amount (positive=right)', required: false },
|
|
292
|
+
deltaY: { type: 'number', description: 'Vertical scroll amount (positive=down)', required: false },
|
|
293
|
+
},
|
|
294
|
+
async execute(args) {
|
|
295
|
+
const result = await client.rpcWithPermissionFlow('desktop.mouse.scroll', {
|
|
296
|
+
x: args.x, y: args.y, deltaX: args.deltaX, deltaY: args.deltaY,
|
|
297
|
+
});
|
|
298
|
+
if (typeof result === 'string') return result;
|
|
299
|
+
return `Scrolled at (${args.x}, ${args.y}) [dx=${args.deltaX || 0}, dy=${args.deltaY || 0}]`;
|
|
300
|
+
},
|
|
301
|
+
},
|
|
302
|
+
|
|
303
|
+
{
|
|
304
|
+
name: 'tunnel_type',
|
|
305
|
+
description: `Type text into the currently focused application on the user's machine via Agent Tunnel.`,
|
|
306
|
+
parameters: {
|
|
307
|
+
tunnel_id: tunnelIdParam,
|
|
308
|
+
text: { type: 'string', description: 'Text to type', required: true },
|
|
309
|
+
delay: { type: 'number', description: 'Delay between characters in ms', required: false },
|
|
310
|
+
},
|
|
311
|
+
async execute(args) {
|
|
312
|
+
const result = await client.rpcWithPermissionFlow('desktop.keyboard.type', {
|
|
313
|
+
text: args.text, delay: args.delay,
|
|
314
|
+
});
|
|
315
|
+
if (typeof result === 'string') return result;
|
|
316
|
+
return `Typed ${(args.text as string).length} characters`;
|
|
317
|
+
},
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
name: 'tunnel_key',
|
|
321
|
+
description: `Press a key combination on the user's machine via Agent Tunnel. Use for keyboard shortcuts like cmd+s, ctrl+c, enter, tab, etc.`,
|
|
322
|
+
parameters: {
|
|
323
|
+
tunnel_id: tunnelIdParam,
|
|
324
|
+
keys: { type: 'array', description: "Keys to press simultaneously. Examples: ['cmd', 's'] for save, ['enter'] for enter", required: true, items: { type: 'string' } },
|
|
325
|
+
},
|
|
326
|
+
async execute(args) {
|
|
327
|
+
const result = await client.rpcWithPermissionFlow('desktop.keyboard.key', { keys: args.keys });
|
|
328
|
+
if (typeof result === 'string') return result;
|
|
329
|
+
return `Pressed: ${(args.keys as string[]).join('+')}`;
|
|
330
|
+
},
|
|
331
|
+
},
|
|
332
|
+
|
|
333
|
+
{
|
|
334
|
+
name: 'tunnel_window_list',
|
|
335
|
+
description: `List all visible windows on the user's machine via Agent Tunnel. Returns window IDs, app names, titles, positions, and sizes.`,
|
|
336
|
+
parameters: {
|
|
337
|
+
tunnel_id: tunnelIdParam,
|
|
338
|
+
},
|
|
339
|
+
async execute() {
|
|
340
|
+
const result = await client.rpcWithPermissionFlow('desktop.window.list', {});
|
|
341
|
+
if (typeof result === 'string') return result;
|
|
342
|
+
|
|
343
|
+
const data = result as { windows: Array<{ id: number; app: string; title: string; bounds: { x: number; y: number; width: number; height: number }; minimized: boolean }> };
|
|
344
|
+
if (data.windows.length === 0) return 'No windows found';
|
|
345
|
+
|
|
346
|
+
const lines = [`=== Windows (${data.windows.length}) ===`];
|
|
347
|
+
for (const w of data.windows) {
|
|
348
|
+
const min = w.minimized ? ' [minimized]' : '';
|
|
349
|
+
lines.push(` #${w.id} | ${w.app} — "${w.title}" | ${w.bounds.x},${w.bounds.y} ${w.bounds.width}x${w.bounds.height}${min}`);
|
|
350
|
+
}
|
|
351
|
+
return lines.join('\n');
|
|
352
|
+
},
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
name: 'tunnel_window_focus',
|
|
356
|
+
description: `Bring a window to the front on the user's machine via Agent Tunnel.`,
|
|
357
|
+
parameters: {
|
|
358
|
+
tunnel_id: tunnelIdParam,
|
|
359
|
+
windowId: { type: 'number', description: 'Window ID from tunnel_window_list', required: true },
|
|
360
|
+
},
|
|
361
|
+
async execute(args) {
|
|
362
|
+
const result = await client.rpcWithPermissionFlow('desktop.window.focus', { windowId: args.windowId });
|
|
363
|
+
if (typeof result === 'string') return result;
|
|
364
|
+
return `Window #${args.windowId} focused`;
|
|
365
|
+
},
|
|
366
|
+
},
|
|
367
|
+
|
|
368
|
+
{
|
|
369
|
+
name: 'tunnel_app_launch',
|
|
370
|
+
description: `Launch an application on the user's machine via Agent Tunnel.`,
|
|
371
|
+
parameters: {
|
|
372
|
+
tunnel_id: tunnelIdParam,
|
|
373
|
+
app: { type: 'string', description: 'Application name to launch', required: true },
|
|
374
|
+
},
|
|
375
|
+
async execute(args) {
|
|
376
|
+
const result = await client.rpcWithPermissionFlow('desktop.app.launch', { app: args.app });
|
|
377
|
+
if (typeof result === 'string') return result;
|
|
378
|
+
return `Launched: ${args.app}`;
|
|
379
|
+
},
|
|
380
|
+
},
|
|
381
|
+
{
|
|
382
|
+
name: 'tunnel_app_quit',
|
|
383
|
+
description: `Quit an application on the user's machine via Agent Tunnel.`,
|
|
384
|
+
parameters: {
|
|
385
|
+
tunnel_id: tunnelIdParam,
|
|
386
|
+
app: { type: 'string', description: 'Application name to quit', required: true },
|
|
387
|
+
},
|
|
388
|
+
async execute(args) {
|
|
389
|
+
const result = await client.rpcWithPermissionFlow('desktop.app.quit', { app: args.app });
|
|
390
|
+
if (typeof result === 'string') return result;
|
|
391
|
+
return `Quit: ${args.app}`;
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
|
|
395
|
+
{
|
|
396
|
+
name: 'tunnel_clipboard_read',
|
|
397
|
+
description: `Read the clipboard contents from the user's machine via Agent Tunnel.`,
|
|
398
|
+
parameters: {
|
|
399
|
+
tunnel_id: tunnelIdParam,
|
|
400
|
+
},
|
|
401
|
+
async execute() {
|
|
402
|
+
const result = await client.rpcWithPermissionFlow('desktop.clipboard.read', {});
|
|
403
|
+
if (typeof result === 'string') return result;
|
|
404
|
+
const data = result as { text: string };
|
|
405
|
+
if (!data.text) return '(clipboard is empty)';
|
|
406
|
+
return `=== Clipboard ===\n${data.text}`;
|
|
407
|
+
},
|
|
408
|
+
},
|
|
409
|
+
{
|
|
410
|
+
name: 'tunnel_clipboard_write',
|
|
411
|
+
description: `Write text to the clipboard on the user's machine via Agent Tunnel.`,
|
|
412
|
+
parameters: {
|
|
413
|
+
tunnel_id: tunnelIdParam,
|
|
414
|
+
text: { type: 'string', description: 'Text to write to clipboard', required: true },
|
|
415
|
+
},
|
|
416
|
+
async execute(args) {
|
|
417
|
+
const result = await client.rpcWithPermissionFlow('desktop.clipboard.write', { text: args.text });
|
|
418
|
+
if (typeof result === 'string') return result;
|
|
419
|
+
return `Clipboard updated (${(args.text as string).length} chars)`;
|
|
420
|
+
},
|
|
421
|
+
},
|
|
422
|
+
|
|
423
|
+
{
|
|
424
|
+
name: 'tunnel_screen_info',
|
|
425
|
+
description: `Get screen resolution and scale factor from the user's machine via Agent Tunnel.`,
|
|
426
|
+
parameters: {
|
|
427
|
+
tunnel_id: tunnelIdParam,
|
|
428
|
+
},
|
|
429
|
+
async execute() {
|
|
430
|
+
const result = await client.rpcWithPermissionFlow('desktop.screen.info', {});
|
|
431
|
+
if (typeof result === 'string') return result;
|
|
432
|
+
const data = result as { width: number; height: number; scaleFactor: number };
|
|
433
|
+
return `Screen: ${data.width}x${data.height} @ ${data.scaleFactor}x scale`;
|
|
434
|
+
},
|
|
435
|
+
},
|
|
436
|
+
|
|
437
|
+
{
|
|
438
|
+
name: 'tunnel_cursor_image',
|
|
439
|
+
description: `Take a small screenshot around the current cursor position on the user's machine via Agent Tunnel.`,
|
|
440
|
+
parameters: {
|
|
441
|
+
tunnel_id: tunnelIdParam,
|
|
442
|
+
radius: { type: 'number', description: 'Radius in pixels around cursor (default: 50)', required: false },
|
|
443
|
+
},
|
|
444
|
+
async execute(args) {
|
|
445
|
+
const result = await client.rpcWithPermissionFlow('desktop.cursor.image', { radius: args.radius });
|
|
446
|
+
if (typeof result === 'string') return result;
|
|
447
|
+
|
|
448
|
+
const data = result as { image: string; width: number; height: number; format?: string };
|
|
449
|
+
const format = data.format || 'png';
|
|
450
|
+
const sizeKB = Math.round(data.image.length * 0.75 / 1024);
|
|
451
|
+
const path = saveImage(data.image, format);
|
|
452
|
+
return `Cursor area saved: ${path}\nDimensions: ${data.width}x${data.height} ${format.toUpperCase()} (${sizeKB}KB)\n\nUse the Read tool to view this image.`;
|
|
453
|
+
},
|
|
454
|
+
},
|
|
455
|
+
|
|
456
|
+
{
|
|
457
|
+
name: 'tunnel_ax_tree',
|
|
458
|
+
description: `Get the accessibility tree of an application on the user's machine via Agent Tunnel. Returns a structured tree of UI elements with roles, labels, states, and available actions.`,
|
|
459
|
+
parameters: {
|
|
460
|
+
tunnel_id: tunnelIdParam,
|
|
461
|
+
pid: { type: 'number', description: 'Process ID of the target application (omit for all apps)', required: false },
|
|
462
|
+
maxDepth: { type: 'number', description: 'Maximum tree depth (default: 8)', required: false },
|
|
463
|
+
roles: { type: 'array', description: "Filter by element roles (e.g., ['button', 'textfield'])", required: false, items: { type: 'string' } },
|
|
464
|
+
},
|
|
465
|
+
async execute(args) {
|
|
466
|
+
const params: Record<string, unknown> = {};
|
|
467
|
+
if (args.pid !== undefined) params.pid = args.pid;
|
|
468
|
+
if (args.maxDepth !== undefined) params.maxDepth = args.maxDepth;
|
|
469
|
+
if (args.roles !== undefined) params.roles = args.roles;
|
|
470
|
+
|
|
471
|
+
const result = await client.rpcWithPermissionFlow('desktop.ax.tree', params);
|
|
472
|
+
if (typeof result === 'string') return result;
|
|
473
|
+
|
|
474
|
+
const data = result as { root: AXElement; elementCount: number };
|
|
475
|
+
if (!data.root) return 'No accessibility tree available';
|
|
476
|
+
|
|
477
|
+
const tree = formatAXTree(data.root);
|
|
478
|
+
return `=== Accessibility Tree (${data.elementCount} elements) ===\n${tree}`;
|
|
479
|
+
},
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
name: 'tunnel_ax_action',
|
|
483
|
+
description: `Perform an accessibility action on a UI element via Agent Tunnel. Returns before/after state to verify the action worked. Common actions: AXPress, AXConfirm, AXCancel, AXRaise, AXShowMenu.`,
|
|
484
|
+
parameters: {
|
|
485
|
+
tunnel_id: tunnelIdParam,
|
|
486
|
+
elementId: { type: 'string', description: "Element ID from the accessibility tree (e.g., '0.3.1')", required: true },
|
|
487
|
+
action: { type: 'string', description: 'Action to perform: AXPress, AXConfirm, AXCancel, AXRaise, AXShowMenu', required: true },
|
|
488
|
+
pid: { type: 'number', description: 'Process ID of the target application', required: false },
|
|
489
|
+
},
|
|
490
|
+
async execute(args) {
|
|
491
|
+
const result = await client.rpcWithPermissionFlow('desktop.ax.action', {
|
|
492
|
+
elementId: args.elementId, action: args.action, pid: args.pid,
|
|
493
|
+
});
|
|
494
|
+
if (typeof result === 'string') return result;
|
|
495
|
+
|
|
496
|
+
const data = result as {
|
|
497
|
+
ok: boolean; action: string; elementId: string;
|
|
498
|
+
before: { focused: boolean; value: string };
|
|
499
|
+
after: { focused: boolean; value: string };
|
|
500
|
+
stateChanged: boolean; role: string; title: string;
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
const lines = [`Action "${data.action}" on [${data.role}] "${data.title}" (${data.elementId})`];
|
|
504
|
+
lines.push(`State changed: ${data.stateChanged ? 'YES' : 'NO'}`);
|
|
505
|
+
lines.push(`Before: focused=${data.before.focused}, value="${data.before.value}"`);
|
|
506
|
+
lines.push(`After: focused=${data.after.focused}, value="${data.after.value}"`);
|
|
507
|
+
if (!data.stateChanged) {
|
|
508
|
+
lines.push('WARNING: No state change detected. The action may not have had any effect.');
|
|
509
|
+
}
|
|
510
|
+
return lines.join('\n');
|
|
511
|
+
},
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
name: 'tunnel_ax_set_value',
|
|
515
|
+
description: `Directly set the value of a UI element (text field, search box, etc.) via the accessibility API. Much more reliable than clicking and typing.`,
|
|
516
|
+
parameters: {
|
|
517
|
+
tunnel_id: tunnelIdParam,
|
|
518
|
+
elementId: { type: 'string', description: "Element ID from the accessibility tree (e.g., '0.3.1')", required: true },
|
|
519
|
+
value: { type: 'string', description: 'The value to set (e.g., text to put in a search field)', required: true },
|
|
520
|
+
pid: { type: 'number', description: 'Process ID of the target application', required: false },
|
|
521
|
+
},
|
|
522
|
+
async execute(args) {
|
|
523
|
+
const result = await client.rpcWithPermissionFlow('desktop.ax.set_value', {
|
|
524
|
+
elementId: args.elementId, value: args.value, pid: args.pid,
|
|
525
|
+
});
|
|
526
|
+
if (typeof result === 'string') return result;
|
|
527
|
+
|
|
528
|
+
const data = result as {
|
|
529
|
+
ok: boolean; elementId: string;
|
|
530
|
+
requestedValue: string; actualValue: string; error?: string;
|
|
531
|
+
};
|
|
532
|
+
|
|
533
|
+
if (data.ok) {
|
|
534
|
+
return `Value set successfully on ${data.elementId}\nRequested: "${data.requestedValue}"\nVerified: "${data.actualValue}"`;
|
|
535
|
+
} else {
|
|
536
|
+
return `FAILED to set value on ${data.elementId}\nRequested: "${data.requestedValue}"\nActual: "${data.actualValue}"\nError: ${data.error || 'unknown'}`;
|
|
537
|
+
}
|
|
538
|
+
},
|
|
539
|
+
},
|
|
540
|
+
{
|
|
541
|
+
name: 'tunnel_ax_focus',
|
|
542
|
+
description: `Focus a UI element directly via the accessibility API. More reliable than clicking to focus.`,
|
|
543
|
+
parameters: {
|
|
544
|
+
tunnel_id: tunnelIdParam,
|
|
545
|
+
elementId: { type: 'string', description: "Element ID from the accessibility tree (e.g., '0.3.1')", required: true },
|
|
546
|
+
pid: { type: 'number', description: 'Process ID of the target application', required: false },
|
|
547
|
+
},
|
|
548
|
+
async execute(args) {
|
|
549
|
+
const result = await client.rpcWithPermissionFlow('desktop.ax.focus', {
|
|
550
|
+
elementId: args.elementId, pid: args.pid,
|
|
551
|
+
});
|
|
552
|
+
if (typeof result === 'string') return result;
|
|
553
|
+
|
|
554
|
+
const data = result as {
|
|
555
|
+
ok: boolean; elementId: string; role: string; title: string;
|
|
556
|
+
before: { focused: boolean }; after: { focused: boolean }; error?: string;
|
|
557
|
+
};
|
|
558
|
+
|
|
559
|
+
if (data.ok) {
|
|
560
|
+
return `Focused [${data.role}] "${data.title}" (${data.elementId})\nBefore: focused=${data.before.focused}\nAfter: focused=${data.after.focused}`;
|
|
561
|
+
} else {
|
|
562
|
+
return `FAILED to focus [${data.role}] "${data.title}" (${data.elementId})\nError: ${data.error || 'unknown'}\nBefore: focused=${data.before.focused}\nAfter: focused=${data.after.focused}`;
|
|
563
|
+
}
|
|
564
|
+
},
|
|
565
|
+
},
|
|
566
|
+
{
|
|
567
|
+
name: 'tunnel_ax_search',
|
|
568
|
+
description: `Search the accessibility tree for UI elements matching a query via Agent Tunnel. Case-insensitive substring match on titles, values, and descriptions.`,
|
|
569
|
+
parameters: {
|
|
570
|
+
tunnel_id: tunnelIdParam,
|
|
571
|
+
query: { type: 'string', description: 'Search text (matches against title, value, description)', required: true },
|
|
572
|
+
role: { type: 'string', description: "Filter by element role (e.g., 'button', 'textfield')", required: false },
|
|
573
|
+
pid: { type: 'number', description: 'Process ID of the target application', required: false },
|
|
574
|
+
maxResults: { type: 'number', description: 'Maximum results to return (default: 20)', required: false },
|
|
575
|
+
},
|
|
576
|
+
async execute(args) {
|
|
577
|
+
const params: Record<string, unknown> = { query: args.query };
|
|
578
|
+
if (args.role !== undefined) params.role = args.role;
|
|
579
|
+
if (args.pid !== undefined) params.pid = args.pid;
|
|
580
|
+
if (args.maxResults !== undefined) params.maxResults = args.maxResults;
|
|
581
|
+
|
|
582
|
+
const result = await client.rpcWithPermissionFlow('desktop.ax.search', params);
|
|
583
|
+
if (typeof result === 'string') return result;
|
|
584
|
+
|
|
585
|
+
const data = result as { elements: AXElement[] };
|
|
586
|
+
if (!data.elements || data.elements.length === 0) return `No elements found matching "${args.query}"`;
|
|
587
|
+
|
|
588
|
+
const lines = [`=== AX Search: "${args.query}" (${data.elements.length} results) ===`];
|
|
589
|
+
for (const el of data.elements) {
|
|
590
|
+
const label = el.title || el.value || el.description || '(unnamed)';
|
|
591
|
+
const flags: string[] = [];
|
|
592
|
+
if (!el.enabled) flags.push('disabled');
|
|
593
|
+
if (el.focused) flags.push('focused');
|
|
594
|
+
if (el.actions.length > 0) flags.push(`actions: ${el.actions.join(',')}`);
|
|
595
|
+
const flagStr = flags.length > 0 ? ` [${flags.join(', ')}]` : '';
|
|
596
|
+
const b = el.bounds;
|
|
597
|
+
lines.push(` [${el.role}] ${label} (id: ${el.id}) @ ${b.x},${b.y} ${b.width}x${b.height}${flagStr}`);
|
|
598
|
+
}
|
|
599
|
+
return lines.join('\n');
|
|
600
|
+
},
|
|
601
|
+
},
|
|
602
|
+
];
|
|
603
|
+
}
|