illuma-agents 1.0.45 → 1.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/deepagents/DeepAgentBackend.cjs +170 -0
- package/dist/cjs/deepagents/DeepAgentBackend.cjs.map +1 -0
- package/dist/cjs/deepagents/DeepAgentRuntime.cjs +135 -0
- package/dist/cjs/deepagents/DeepAgentRuntime.cjs.map +1 -0
- package/dist/cjs/deepagents/types.cjs +29 -0
- package/dist/cjs/deepagents/types.cjs.map +1 -0
- package/dist/cjs/main.cjs +17 -6
- package/dist/cjs/main.cjs.map +1 -1
- package/dist/esm/deepagents/DeepAgentBackend.mjs +168 -0
- package/dist/esm/deepagents/DeepAgentBackend.mjs.map +1 -0
- package/dist/esm/deepagents/DeepAgentRuntime.mjs +132 -0
- package/dist/esm/deepagents/DeepAgentRuntime.mjs.map +1 -0
- package/dist/esm/deepagents/types.mjs +26 -0
- package/dist/esm/deepagents/types.mjs.map +1 -0
- package/dist/esm/main.mjs +4 -1
- package/dist/esm/main.mjs.map +1 -1
- package/dist/types/deepagents/DeepAgentBackend.d.ts +82 -0
- package/dist/types/deepagents/DeepAgentRuntime.d.ts +46 -0
- package/dist/types/deepagents/index.d.ts +16 -0
- package/dist/types/deepagents/types.d.ts +105 -0
- package/dist/types/index.d.ts +1 -1
- package/package.json +2 -1
- package/src/deepagents/DeepAgentBackend.ts +214 -0
- package/src/deepagents/DeepAgentRuntime.ts +187 -0
- package/src/deepagents/index.ts +25 -0
- package/src/deepagents/types.ts +118 -0
- package/src/index.ts +3 -1
- package/src/specs/deepagents.test.ts +286 -0
- package/dist/cjs/tools/DesktopTools.cjs +0 -363
- package/dist/cjs/tools/DesktopTools.cjs.map +0 -1
- package/dist/esm/tools/DesktopTools.mjs +0 -357
- package/dist/esm/tools/DesktopTools.mjs.map +0 -1
- package/dist/types/tools/DesktopTools.d.ts +0 -134
- package/src/tools/DesktopTools.ts +0 -574
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep Agents Unit Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests for the deep agents integration module
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
DeepAgentBackend,
|
|
9
|
+
calculateTodoProgress,
|
|
10
|
+
defaultDeepAgentState,
|
|
11
|
+
type DeepAgentTodo,
|
|
12
|
+
type DeepAgentSubagent,
|
|
13
|
+
type DeepAgentState,
|
|
14
|
+
type TodoStatus,
|
|
15
|
+
} from '../deepagents';
|
|
16
|
+
|
|
17
|
+
describe('Deep Agents Types', () => {
|
|
18
|
+
describe('calculateTodoProgress', () => {
|
|
19
|
+
it('should return 0 for empty todos', () => {
|
|
20
|
+
expect(calculateTodoProgress([])).toBe(0);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it('should return 0 when no todos are completed', () => {
|
|
24
|
+
const todos: DeepAgentTodo[] = [
|
|
25
|
+
{ id: '1', content: 'Task 1', status: 'pending' },
|
|
26
|
+
{ id: '2', content: 'Task 2', status: 'in_progress' },
|
|
27
|
+
];
|
|
28
|
+
expect(calculateTodoProgress(todos)).toBe(0);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it('should return 50 when half of todos are completed', () => {
|
|
32
|
+
const todos: DeepAgentTodo[] = [
|
|
33
|
+
{ id: '1', content: 'Task 1', status: 'completed' },
|
|
34
|
+
{ id: '2', content: 'Task 2', status: 'pending' },
|
|
35
|
+
];
|
|
36
|
+
expect(calculateTodoProgress(todos)).toBe(50);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('should return 100 when all todos are completed', () => {
|
|
40
|
+
const todos: DeepAgentTodo[] = [
|
|
41
|
+
{ id: '1', content: 'Task 1', status: 'completed' },
|
|
42
|
+
{ id: '2', content: 'Task 2', status: 'completed' },
|
|
43
|
+
{ id: '3', content: 'Task 3', status: 'completed' },
|
|
44
|
+
];
|
|
45
|
+
expect(calculateTodoProgress(todos)).toBe(100);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('should round progress correctly', () => {
|
|
49
|
+
const todos: DeepAgentTodo[] = [
|
|
50
|
+
{ id: '1', content: 'Task 1', status: 'completed' },
|
|
51
|
+
{ id: '2', content: 'Task 2', status: 'pending' },
|
|
52
|
+
{ id: '3', content: 'Task 3', status: 'pending' },
|
|
53
|
+
];
|
|
54
|
+
expect(calculateTodoProgress(todos)).toBe(33); // 33.33... rounds to 33
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
describe('defaultDeepAgentState', () => {
|
|
59
|
+
it('should have correct default values', () => {
|
|
60
|
+
expect(defaultDeepAgentState).toEqual({
|
|
61
|
+
enabled: false,
|
|
62
|
+
todos: [],
|
|
63
|
+
subagents: [],
|
|
64
|
+
progress: 0,
|
|
65
|
+
isRunning: false,
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('should be immutable (defensive copy check)', () => {
|
|
70
|
+
const state = { ...defaultDeepAgentState };
|
|
71
|
+
state.enabled = true;
|
|
72
|
+
expect(defaultDeepAgentState.enabled).toBe(false);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe('TodoStatus type', () => {
|
|
77
|
+
it('should accept valid status values', () => {
|
|
78
|
+
const statuses: TodoStatus[] = ['pending', 'in_progress', 'completed', 'cancelled'];
|
|
79
|
+
expect(statuses).toHaveLength(4);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe('DeepAgentTodo interface', () => {
|
|
84
|
+
it('should accept minimal todo', () => {
|
|
85
|
+
const todo: DeepAgentTodo = {
|
|
86
|
+
id: '1',
|
|
87
|
+
content: 'Test task',
|
|
88
|
+
status: 'pending',
|
|
89
|
+
};
|
|
90
|
+
expect(todo.id).toBe('1');
|
|
91
|
+
expect(todo.content).toBe('Test task');
|
|
92
|
+
expect(todo.status).toBe('pending');
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('should accept todo with optional fields', () => {
|
|
96
|
+
const todo: DeepAgentTodo = {
|
|
97
|
+
id: '1',
|
|
98
|
+
content: 'Test task',
|
|
99
|
+
status: 'completed',
|
|
100
|
+
order: 0,
|
|
101
|
+
createdAt: '2026-01-25T00:00:00Z',
|
|
102
|
+
updatedAt: '2026-01-25T01:00:00Z',
|
|
103
|
+
};
|
|
104
|
+
expect(todo.order).toBe(0);
|
|
105
|
+
expect(todo.createdAt).toBeDefined();
|
|
106
|
+
expect(todo.updatedAt).toBeDefined();
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe('DeepAgentSubagent interface', () => {
|
|
111
|
+
it('should accept minimal subagent', () => {
|
|
112
|
+
const subagent: DeepAgentSubagent = {
|
|
113
|
+
id: '1',
|
|
114
|
+
name: 'Research Agent',
|
|
115
|
+
status: 'pending',
|
|
116
|
+
};
|
|
117
|
+
expect(subagent.id).toBe('1');
|
|
118
|
+
expect(subagent.name).toBe('Research Agent');
|
|
119
|
+
expect(subagent.status).toBe('pending');
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('should accept subagent with optional fields', () => {
|
|
123
|
+
const subagent: DeepAgentSubagent = {
|
|
124
|
+
id: '1',
|
|
125
|
+
name: 'Research Agent',
|
|
126
|
+
description: 'Researches topics',
|
|
127
|
+
status: 'completed',
|
|
128
|
+
startedAt: '2026-01-25T00:00:00Z',
|
|
129
|
+
completedAt: '2026-01-25T01:00:00Z',
|
|
130
|
+
};
|
|
131
|
+
expect(subagent.description).toBe('Researches topics');
|
|
132
|
+
expect(subagent.startedAt).toBeDefined();
|
|
133
|
+
expect(subagent.completedAt).toBeDefined();
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
describe('DeepAgentBackend', () => {
|
|
139
|
+
describe('constructor', () => {
|
|
140
|
+
it('should create backend with default options', () => {
|
|
141
|
+
const backend = new DeepAgentBackend();
|
|
142
|
+
expect(backend.id).toMatch(/^deep-agent-backend-/);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('should create backend with custom options', () => {
|
|
146
|
+
const backend = new DeepAgentBackend({
|
|
147
|
+
rootDir: '/tmp/test',
|
|
148
|
+
virtualMode: true,
|
|
149
|
+
timeout: 60000,
|
|
150
|
+
maxOutputBytes: 50000,
|
|
151
|
+
conversationId: 'test-convo',
|
|
152
|
+
});
|
|
153
|
+
expect(backend.id).toMatch(/^deep-agent-backend-/);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
describe('execute', () => {
|
|
158
|
+
it('should return error for empty command', async () => {
|
|
159
|
+
const backend = new DeepAgentBackend();
|
|
160
|
+
const result = await backend.execute('');
|
|
161
|
+
expect(result.exitCode).toBe(1);
|
|
162
|
+
expect(result.output).toContain('Error');
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it('should return warning when no code executor configured', async () => {
|
|
166
|
+
const backend = new DeepAgentBackend();
|
|
167
|
+
const result = await backend.execute('echo test');
|
|
168
|
+
expect(result.exitCode).toBe(1);
|
|
169
|
+
expect(result.output).toContain('not available');
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
describe('sendEvent', () => {
|
|
174
|
+
it('should send event when SSE response is available', () => {
|
|
175
|
+
const writeData: string[] = [];
|
|
176
|
+
const mockSseResponse = {
|
|
177
|
+
write: (data: string) => writeData.push(data),
|
|
178
|
+
headersSent: false,
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
const backend = new DeepAgentBackend({
|
|
182
|
+
sseResponse: mockSseResponse,
|
|
183
|
+
conversationId: 'test-convo',
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
backend.sendEvent('test_event', { foo: 'bar' });
|
|
187
|
+
|
|
188
|
+
expect(writeData.length).toBe(1);
|
|
189
|
+
expect(writeData[0]).toContain('event: test_event');
|
|
190
|
+
expect(writeData[0]).toContain('"foo":"bar"');
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it('should not throw when SSE response is not available', () => {
|
|
194
|
+
const backend = new DeepAgentBackend();
|
|
195
|
+
expect(() => backend.sendEvent('test', {})).not.toThrow();
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
describe('emitTodoUpdate', () => {
|
|
200
|
+
it('should emit todo update event', () => {
|
|
201
|
+
const writeData: string[] = [];
|
|
202
|
+
const mockSseResponse = {
|
|
203
|
+
write: (data: string) => writeData.push(data),
|
|
204
|
+
headersSent: false,
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
const todos: DeepAgentTodo[] = [
|
|
208
|
+
{ id: '1', content: 'Task 1', status: 'completed' },
|
|
209
|
+
{ id: '2', content: 'Task 2', status: 'pending' },
|
|
210
|
+
];
|
|
211
|
+
|
|
212
|
+
const backend = new DeepAgentBackend({
|
|
213
|
+
sseResponse: mockSseResponse,
|
|
214
|
+
conversationId: 'test-convo',
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
backend.emitTodoUpdate(todos);
|
|
218
|
+
|
|
219
|
+
expect(writeData.length).toBe(1);
|
|
220
|
+
expect(writeData[0]).toContain('event: deep_research');
|
|
221
|
+
expect(writeData[0]).toContain('todo_update');
|
|
222
|
+
expect(writeData[0]).toContain('test-convo');
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('should call onTodoUpdate callback', () => {
|
|
226
|
+
const receivedTodos: DeepAgentTodo[][] = [];
|
|
227
|
+
const onTodoUpdate = (todos: DeepAgentTodo[]) => receivedTodos.push(todos);
|
|
228
|
+
|
|
229
|
+
const todos: DeepAgentTodo[] = [
|
|
230
|
+
{ id: '1', content: 'Task 1', status: 'pending' },
|
|
231
|
+
];
|
|
232
|
+
|
|
233
|
+
const backend = new DeepAgentBackend({
|
|
234
|
+
onTodoUpdate,
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
backend.emitTodoUpdate(todos);
|
|
238
|
+
|
|
239
|
+
expect(receivedTodos.length).toBe(1);
|
|
240
|
+
expect(receivedTodos[0]).toEqual(todos);
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
describe('emitSubagentUpdate', () => {
|
|
245
|
+
it('should emit subagent update event', () => {
|
|
246
|
+
const writeData: string[] = [];
|
|
247
|
+
const mockSseResponse = {
|
|
248
|
+
write: (data: string) => writeData.push(data),
|
|
249
|
+
headersSent: false,
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
const subagents: DeepAgentSubagent[] = [
|
|
253
|
+
{ id: '1', name: 'Agent 1', status: 'running' },
|
|
254
|
+
];
|
|
255
|
+
|
|
256
|
+
const backend = new DeepAgentBackend({
|
|
257
|
+
sseResponse: mockSseResponse,
|
|
258
|
+
conversationId: 'test-convo',
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
backend.emitSubagentUpdate(subagents);
|
|
262
|
+
|
|
263
|
+
expect(writeData.length).toBe(1);
|
|
264
|
+
expect(writeData[0]).toContain('event: deep_research');
|
|
265
|
+
expect(writeData[0]).toContain('subagent_update');
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
it('should call onSubagentUpdate callback', () => {
|
|
269
|
+
const receivedSubagents: DeepAgentSubagent[][] = [];
|
|
270
|
+
const onSubagentUpdate = (subagents: DeepAgentSubagent[]) => receivedSubagents.push(subagents);
|
|
271
|
+
|
|
272
|
+
const subagents: DeepAgentSubagent[] = [
|
|
273
|
+
{ id: '1', name: 'Agent 1', status: 'running' },
|
|
274
|
+
];
|
|
275
|
+
|
|
276
|
+
const backend = new DeepAgentBackend({
|
|
277
|
+
onSubagentUpdate,
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
backend.emitSubagentUpdate(subagents);
|
|
281
|
+
|
|
282
|
+
expect(receivedSubagents.length).toBe(1);
|
|
283
|
+
expect(receivedSubagents[0]).toEqual(subagents);
|
|
284
|
+
});
|
|
285
|
+
});
|
|
286
|
+
});
|
|
@@ -1,363 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var zod = require('zod');
|
|
4
|
-
var tools = require('@langchain/core/tools');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Desktop tool names - keep in sync with Ranger Desktop Electron app
|
|
8
|
-
* These tools execute locally in the Electron app, NOT on the server
|
|
9
|
-
*/
|
|
10
|
-
const EDesktopTools = {
|
|
11
|
-
SCREENSHOT: 'computer_screenshot',
|
|
12
|
-
CLICK: 'computer_click',
|
|
13
|
-
DOUBLE_CLICK: 'computer_double_click',
|
|
14
|
-
RIGHT_CLICK: 'computer_right_click',
|
|
15
|
-
TYPE: 'computer_type',
|
|
16
|
-
KEY: 'computer_key',
|
|
17
|
-
KEY_COMBO: 'computer_key_combo',
|
|
18
|
-
SCROLL: 'computer_scroll',
|
|
19
|
-
DRAG: 'computer_drag',
|
|
20
|
-
GET_ACTIVE_WINDOW: 'computer_get_active_window',
|
|
21
|
-
GET_MOUSE_POSITION: 'computer_get_mouse_position',
|
|
22
|
-
CLIPBOARD_READ: 'clipboard_read',
|
|
23
|
-
CLIPBOARD_WRITE: 'clipboard_write',
|
|
24
|
-
CLIPBOARD_PASTE: 'clipboard_paste',
|
|
25
|
-
WAIT: 'computer_wait',
|
|
26
|
-
// Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
|
|
27
|
-
UI_FIND_ELEMENT: 'ui_find_element',
|
|
28
|
-
UI_CLICK_ELEMENT: 'ui_click_element',
|
|
29
|
-
UI_GET_WINDOW_TREE: 'ui_get_window_tree',
|
|
30
|
-
UI_FIND_BUTTONS: 'ui_find_buttons',
|
|
31
|
-
UI_FIND_INPUTS: 'ui_find_inputs',
|
|
32
|
-
};
|
|
33
|
-
/**
|
|
34
|
-
* Check if desktop capability is available based on request headers or context
|
|
35
|
-
* The Ranger Desktop Electron app sets these headers when connected:
|
|
36
|
-
* - X-Ranger-Desktop: true
|
|
37
|
-
* - X-Ranger-Desktop-Capable: true
|
|
38
|
-
*/
|
|
39
|
-
function hasDesktopCapability(req) {
|
|
40
|
-
if (!req?.headers) {
|
|
41
|
-
return false;
|
|
42
|
-
}
|
|
43
|
-
const desktopApp = req.headers['x-ranger-desktop'];
|
|
44
|
-
const desktopCapable = req.headers['x-ranger-desktop-capable'];
|
|
45
|
-
return desktopApp === 'true' || desktopCapable === 'true';
|
|
46
|
-
}
|
|
47
|
-
// Tool schemas
|
|
48
|
-
const ScreenshotSchema = zod.z.object({});
|
|
49
|
-
const ClickSchema = zod.z.object({
|
|
50
|
-
x: zod.z.number().describe('X coordinate to click'),
|
|
51
|
-
y: zod.z.number().describe('Y coordinate to click'),
|
|
52
|
-
});
|
|
53
|
-
const DoubleClickSchema = zod.z.object({
|
|
54
|
-
x: zod.z.number().describe('X coordinate to double-click'),
|
|
55
|
-
y: zod.z.number().describe('Y coordinate to double-click'),
|
|
56
|
-
});
|
|
57
|
-
const RightClickSchema = zod.z.object({
|
|
58
|
-
x: zod.z.number().describe('X coordinate to right-click'),
|
|
59
|
-
y: zod.z.number().describe('Y coordinate to right-click'),
|
|
60
|
-
});
|
|
61
|
-
const TypeSchema = zod.z.object({
|
|
62
|
-
text: zod.z.string().describe('Text to type'),
|
|
63
|
-
});
|
|
64
|
-
const KeySchema = zod.z.object({
|
|
65
|
-
key: zod.z
|
|
66
|
-
.string()
|
|
67
|
-
.describe('Key to press (e.g., "Enter", "Tab", "Escape", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "F1"-"F12")'),
|
|
68
|
-
});
|
|
69
|
-
const KeyComboSchema = zod.z.object({
|
|
70
|
-
keys: zod.z
|
|
71
|
-
.array(zod.z.string())
|
|
72
|
-
.describe('Array of keys to press together (e.g., ["Control", "c"] for copy, ["Alt", "Tab"] for window switch)'),
|
|
73
|
-
});
|
|
74
|
-
const ScrollSchema = zod.z.object({
|
|
75
|
-
x: zod.z.number().describe('X coordinate to scroll at'),
|
|
76
|
-
y: zod.z.number().describe('Y coordinate to scroll at'),
|
|
77
|
-
deltaX: zod.z.number().optional().describe('Horizontal scroll amount (pixels)'),
|
|
78
|
-
deltaY: zod.z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),
|
|
79
|
-
});
|
|
80
|
-
const DragSchema = zod.z.object({
|
|
81
|
-
startX: zod.z.number().describe('Starting X coordinate'),
|
|
82
|
-
startY: zod.z.number().describe('Starting Y coordinate'),
|
|
83
|
-
endX: zod.z.number().describe('Ending X coordinate'),
|
|
84
|
-
endY: zod.z.number().describe('Ending Y coordinate'),
|
|
85
|
-
});
|
|
86
|
-
const GetActiveWindowSchema = zod.z.object({});
|
|
87
|
-
const GetMousePositionSchema = zod.z.object({});
|
|
88
|
-
const ClipboardReadSchema = zod.z.object({});
|
|
89
|
-
const ClipboardWriteSchema = zod.z.object({
|
|
90
|
-
text: zod.z.string().describe('Text to write to clipboard'),
|
|
91
|
-
});
|
|
92
|
-
const ClipboardPasteSchema = zod.z.object({});
|
|
93
|
-
const WaitSchema = zod.z.object({
|
|
94
|
-
ms: zod.z.number().describe('Milliseconds to wait'),
|
|
95
|
-
});
|
|
96
|
-
// ============ Native UI Automation Schemas (Windows) ============
|
|
97
|
-
const UIFindElementSchema = zod.z.object({
|
|
98
|
-
name: zod.z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
|
|
99
|
-
automationId: zod.z.string().optional().describe('Automation ID of element (unique identifier)'),
|
|
100
|
-
controlType: zod.z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
|
|
101
|
-
});
|
|
102
|
-
const UIClickElementSchema = zod.z.object({
|
|
103
|
-
name: zod.z.string().optional().describe('Element name/label to click'),
|
|
104
|
-
automationId: zod.z.string().optional().describe('Automation ID of element to click'),
|
|
105
|
-
controlType: zod.z.string().optional().describe('Type of control: Button, Edit, etc.'),
|
|
106
|
-
clickType: zod.z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
|
|
107
|
-
});
|
|
108
|
-
const UIGetWindowTreeSchema = zod.z.object({
|
|
109
|
-
maxDepth: zod.z.number().optional().describe('Maximum depth to traverse (default: 3)'),
|
|
110
|
-
});
|
|
111
|
-
const UIFindButtonsSchema = zod.z.object({});
|
|
112
|
-
const UIFindInputsSchema = zod.z.object({});
|
|
113
|
-
/**
|
|
114
|
-
* Format desktop action result for LLM consumption
|
|
115
|
-
*/
|
|
116
|
-
function formatResultForLLM(result, action) {
|
|
117
|
-
if (!result.success && result.error) {
|
|
118
|
-
return `Desktop action "${action}" failed: ${result.error}`;
|
|
119
|
-
}
|
|
120
|
-
const parts = [];
|
|
121
|
-
if (result.screenshot) {
|
|
122
|
-
parts.push(`Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`);
|
|
123
|
-
// The base64 image will be handled separately by the message formatter
|
|
124
|
-
}
|
|
125
|
-
if (result.activeWindow) {
|
|
126
|
-
parts.push(`**Active Window:**`);
|
|
127
|
-
parts.push(` - Title: ${result.activeWindow.title}`);
|
|
128
|
-
parts.push(` - App: ${result.activeWindow.app}`);
|
|
129
|
-
if (result.activeWindow.bounds) {
|
|
130
|
-
const b = result.activeWindow.bounds;
|
|
131
|
-
parts.push(` - Position: (${b.x}, ${b.y})`);
|
|
132
|
-
parts.push(` - Size: ${b.width}x${b.height}`);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
if (result.mousePosition) {
|
|
136
|
-
parts.push(`**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`);
|
|
137
|
-
}
|
|
138
|
-
if (result.clipboard !== undefined) {
|
|
139
|
-
parts.push(`**Clipboard Content:** ${result.clipboard}`);
|
|
140
|
-
}
|
|
141
|
-
// UI Automation results
|
|
142
|
-
if (result.uiElement) {
|
|
143
|
-
const el = result.uiElement;
|
|
144
|
-
parts.push(`**UI Element Found:**`);
|
|
145
|
-
parts.push(` - Name: "${el.name}"`);
|
|
146
|
-
parts.push(` - AutomationId: "${el.automationId}"`);
|
|
147
|
-
parts.push(` - Type: ${el.controlType}`);
|
|
148
|
-
if (el.boundingRectangle) {
|
|
149
|
-
const b = el.boundingRectangle;
|
|
150
|
-
parts.push(` - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
|
|
151
|
-
parts.push(` - Center: (${Math.round(b.x + b.width / 2)}, ${Math.round(b.y + b.height / 2)})`);
|
|
152
|
-
}
|
|
153
|
-
parts.push(` - Enabled: ${el.isEnabled}`);
|
|
154
|
-
}
|
|
155
|
-
if (result.uiElements && result.uiElements.length > 0) {
|
|
156
|
-
parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
|
|
157
|
-
for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
|
|
158
|
-
const bounds = el.boundingRectangle ?
|
|
159
|
-
` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
|
|
160
|
-
parts.push(` - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
|
|
161
|
-
}
|
|
162
|
-
if (result.uiElements.length > 20) {
|
|
163
|
-
parts.push(` ... and ${result.uiElements.length - 20} more`);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
if (result.uiTree) {
|
|
167
|
-
parts.push(`**UI Tree:**`);
|
|
168
|
-
parts.push('```json');
|
|
169
|
-
parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
|
|
170
|
-
parts.push('```');
|
|
171
|
-
}
|
|
172
|
-
if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
|
|
173
|
-
// Generic data fallback
|
|
174
|
-
parts.push(`**Result:**`);
|
|
175
|
-
parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
|
|
176
|
-
}
|
|
177
|
-
if (parts.length === 0) {
|
|
178
|
-
parts.push(`Desktop action "${action}" completed successfully.`);
|
|
179
|
-
}
|
|
180
|
-
return parts.join('\\n');
|
|
181
|
-
}
|
|
182
|
-
/**
|
|
183
|
-
* Create desktop automation tools for the agent
|
|
184
|
-
* These tools allow AI to control the user's desktop when Ranger Desktop is running
|
|
185
|
-
*/
|
|
186
|
-
function createDesktopTools(options = {}) {
|
|
187
|
-
const { waitForResult } = options;
|
|
188
|
-
const tools$1 = [];
|
|
189
|
-
/**
|
|
190
|
-
* Helper to create tool function that optionally waits for results
|
|
191
|
-
* The toolCallId is extracted from the RunnableConfig passed by LangChain
|
|
192
|
-
*/
|
|
193
|
-
const createToolFunction = (action) => {
|
|
194
|
-
return async (args, config) => {
|
|
195
|
-
const toolCallId = config?.toolCall?.id ??
|
|
196
|
-
`desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;
|
|
197
|
-
// Create marker for Electron app
|
|
198
|
-
const marker = {
|
|
199
|
-
requiresDesktopExecution: true,
|
|
200
|
-
action,
|
|
201
|
-
args,
|
|
202
|
-
toolCallId,
|
|
203
|
-
};
|
|
204
|
-
// If no callback, return marker immediately (Electron handles via SSE interception)
|
|
205
|
-
if (!waitForResult) {
|
|
206
|
-
return JSON.stringify(marker);
|
|
207
|
-
}
|
|
208
|
-
// With callback: wait for actual results from Electron app
|
|
209
|
-
try {
|
|
210
|
-
const result = await waitForResult(action, args, toolCallId);
|
|
211
|
-
return formatResultForLLM(result, action);
|
|
212
|
-
}
|
|
213
|
-
catch (error) {
|
|
214
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
215
|
-
return `Desktop action "${action}" failed: ${errorMessage}`;
|
|
216
|
-
}
|
|
217
|
-
};
|
|
218
|
-
};
|
|
219
|
-
// computer_screenshot
|
|
220
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.SCREENSHOT), {
|
|
221
|
-
name: EDesktopTools.SCREENSHOT,
|
|
222
|
-
description: 'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
|
|
223
|
-
schema: ScreenshotSchema,
|
|
224
|
-
}));
|
|
225
|
-
// computer_click
|
|
226
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.CLICK), {
|
|
227
|
-
name: EDesktopTools.CLICK,
|
|
228
|
-
description: 'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
|
|
229
|
-
schema: ClickSchema,
|
|
230
|
-
}));
|
|
231
|
-
// computer_double_click
|
|
232
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {
|
|
233
|
-
name: EDesktopTools.DOUBLE_CLICK,
|
|
234
|
-
description: 'Double-click the mouse at the specified screen coordinates.',
|
|
235
|
-
schema: DoubleClickSchema,
|
|
236
|
-
}));
|
|
237
|
-
// computer_right_click
|
|
238
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {
|
|
239
|
-
name: EDesktopTools.RIGHT_CLICK,
|
|
240
|
-
description: 'Right-click the mouse at the specified screen coordinates to open context menus.',
|
|
241
|
-
schema: RightClickSchema,
|
|
242
|
-
}));
|
|
243
|
-
// computer_type
|
|
244
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.TYPE), {
|
|
245
|
-
name: EDesktopTools.TYPE,
|
|
246
|
-
description: 'Type text using the keyboard. Make sure the target input field is focused first (use click).',
|
|
247
|
-
schema: TypeSchema,
|
|
248
|
-
}));
|
|
249
|
-
// computer_key
|
|
250
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.KEY), {
|
|
251
|
-
name: EDesktopTools.KEY,
|
|
252
|
-
description: 'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
|
|
253
|
-
schema: KeySchema,
|
|
254
|
-
}));
|
|
255
|
-
// computer_key_combo
|
|
256
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.KEY_COMBO), {
|
|
257
|
-
name: EDesktopTools.KEY_COMBO,
|
|
258
|
-
description: 'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
|
|
259
|
-
schema: KeyComboSchema,
|
|
260
|
-
}));
|
|
261
|
-
// computer_scroll
|
|
262
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.SCROLL), {
|
|
263
|
-
name: EDesktopTools.SCROLL,
|
|
264
|
-
description: 'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
|
|
265
|
-
schema: ScrollSchema,
|
|
266
|
-
}));
|
|
267
|
-
// computer_drag
|
|
268
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.DRAG), {
|
|
269
|
-
name: EDesktopTools.DRAG,
|
|
270
|
-
description: 'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
|
|
271
|
-
schema: DragSchema,
|
|
272
|
-
}));
|
|
273
|
-
// computer_get_active_window
|
|
274
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {
|
|
275
|
-
name: EDesktopTools.GET_ACTIVE_WINDOW,
|
|
276
|
-
description: 'Get information about the currently active window (title, application name, position, size).',
|
|
277
|
-
schema: GetActiveWindowSchema,
|
|
278
|
-
}));
|
|
279
|
-
// computer_get_mouse_position
|
|
280
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {
|
|
281
|
-
name: EDesktopTools.GET_MOUSE_POSITION,
|
|
282
|
-
description: 'Get the current mouse cursor position on screen.',
|
|
283
|
-
schema: GetMousePositionSchema,
|
|
284
|
-
}));
|
|
285
|
-
// clipboard_read
|
|
286
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {
|
|
287
|
-
name: EDesktopTools.CLIPBOARD_READ,
|
|
288
|
-
description: 'Read the current contents of the system clipboard.',
|
|
289
|
-
schema: ClipboardReadSchema,
|
|
290
|
-
}));
|
|
291
|
-
// clipboard_write
|
|
292
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {
|
|
293
|
-
name: EDesktopTools.CLIPBOARD_WRITE,
|
|
294
|
-
description: 'Write text to the system clipboard.',
|
|
295
|
-
schema: ClipboardWriteSchema,
|
|
296
|
-
}));
|
|
297
|
-
// clipboard_paste
|
|
298
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {
|
|
299
|
-
name: EDesktopTools.CLIPBOARD_PASTE,
|
|
300
|
-
description: 'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
|
|
301
|
-
schema: ClipboardPasteSchema,
|
|
302
|
-
}));
|
|
303
|
-
// computer_wait
|
|
304
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.WAIT), {
|
|
305
|
-
name: EDesktopTools.WAIT,
|
|
306
|
-
description: 'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
|
|
307
|
-
schema: WaitSchema,
|
|
308
|
-
}));
|
|
309
|
-
// ============ Native UI Automation Tools (Windows) ============
|
|
310
|
-
// These are FASTER and MORE RELIABLE than screenshot-based automation
|
|
311
|
-
// They find elements by semantic properties (name, automationId, type)
|
|
312
|
-
// instead of relying on pixel coordinates from screenshots
|
|
313
|
-
// ui_find_element
|
|
314
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
|
|
315
|
-
name: EDesktopTools.UI_FIND_ELEMENT,
|
|
316
|
-
description: '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
|
|
317
|
-
schema: UIFindElementSchema,
|
|
318
|
-
}));
|
|
319
|
-
// ui_click_element
|
|
320
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
|
|
321
|
-
name: EDesktopTools.UI_CLICK_ELEMENT,
|
|
322
|
-
description: '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
|
|
323
|
-
schema: UIClickElementSchema,
|
|
324
|
-
}));
|
|
325
|
-
// ui_get_window_tree
|
|
326
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
|
|
327
|
-
name: EDesktopTools.UI_GET_WINDOW_TREE,
|
|
328
|
-
description: 'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
|
|
329
|
-
schema: UIGetWindowTreeSchema,
|
|
330
|
-
}));
|
|
331
|
-
// ui_find_buttons
|
|
332
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
|
|
333
|
-
name: EDesktopTools.UI_FIND_BUTTONS,
|
|
334
|
-
description: 'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
|
|
335
|
-
schema: UIFindButtonsSchema,
|
|
336
|
-
}));
|
|
337
|
-
// ui_find_inputs
|
|
338
|
-
tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
|
|
339
|
-
name: EDesktopTools.UI_FIND_INPUTS,
|
|
340
|
-
description: 'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
|
|
341
|
-
schema: UIFindInputsSchema,
|
|
342
|
-
}));
|
|
343
|
-
return tools$1;
|
|
344
|
-
}
|
|
345
|
-
/**
|
|
346
|
-
* Get all desktop tool names
|
|
347
|
-
*/
|
|
348
|
-
function getDesktopToolNames() {
|
|
349
|
-
return Object.values(EDesktopTools);
|
|
350
|
-
}
|
|
351
|
-
/**
|
|
352
|
-
* Check if a tool name is a desktop tool
|
|
353
|
-
*/
|
|
354
|
-
function isDesktopTool(name) {
|
|
355
|
-
return Object.values(EDesktopTools).includes(name);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
exports.EDesktopTools = EDesktopTools;
|
|
359
|
-
exports.createDesktopTools = createDesktopTools;
|
|
360
|
-
exports.getDesktopToolNames = getDesktopToolNames;
|
|
361
|
-
exports.hasDesktopCapability = hasDesktopCapability;
|
|
362
|
-
exports.isDesktopTool = isDesktopTool;
|
|
363
|
-
//# sourceMappingURL=DesktopTools.cjs.map
|