@lardexx/arenacode 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -22
- package/package.json +8 -4
- package/src/automation/ffi.js +62 -0
- package/src/automation/linux.js +771 -0
- package/src/automation/mac.js +657 -0
- package/src/automation/png.js +47 -0
- package/src/automation/win-dll.js +493 -0
- package/src/automation.js +88 -0
- package/src/cli.js +5 -2
- package/src/config.js +1 -0
- package/src/local.js +106 -11
- package/src/tui/app.js +40 -2
- package/vendor/windows-use/windows-use.dll +0 -0
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Windows backend: the client uses the same DLL the desktop app uses — `windows-use`
|
|
4
|
+
// (exports wu_call_json / wu_free / wu_version) — through koffi, so both clients run
|
|
5
|
+
// literally the same automation code. If the DLL is not found next to the client and
|
|
6
|
+
// the ArenaCode executable is installed, the DLL is extracted from its resources.
|
|
7
|
+
|
|
8
|
+
const fs = require('node:fs');
|
|
9
|
+
const os = require('node:os');
|
|
10
|
+
const path = require('node:path');
|
|
11
|
+
const { requireFfi } = require('./ffi');
|
|
12
|
+
|
|
13
|
+
const WU_EXPORTS = ['wu_call_json', 'wu_free', 'wu_version'];
|
|
14
|
+
|
|
15
|
+
// ------------------------------------------------------------------ PE reading
|
|
16
|
+
// Small PE reader: only what is needed to walk the resource directory and pull out an
|
|
17
|
+
// RCDATA blob (the DLL is stored there, RCDATA/1002, in the ArenaCode executable).
|
|
18
|
+
|
|
19
|
+
function readResourceBlobs(buffer) {
|
|
20
|
+
if (!Buffer.isBuffer(buffer) || buffer.length < 0x100) return [];
|
|
21
|
+
if (buffer.readUInt16LE(0) !== 0x5a4d) return []; // MZ
|
|
22
|
+
const peOffset = buffer.readUInt32LE(0x3c);
|
|
23
|
+
if (peOffset + 0x18 > buffer.length) return [];
|
|
24
|
+
if (buffer.readUInt32LE(peOffset) !== 0x00004550) return []; // PE\0\0
|
|
25
|
+
const coff = peOffset + 4;
|
|
26
|
+
const numberOfSections = buffer.readUInt16LE(coff + 2);
|
|
27
|
+
const sizeOfOptionalHeader = buffer.readUInt16LE(coff + 16);
|
|
28
|
+
const optional = coff + 20;
|
|
29
|
+
const magic = buffer.readUInt16LE(optional);
|
|
30
|
+
const isPe32Plus = magic === 0x20b;
|
|
31
|
+
const dirOffset = optional + (isPe32Plus ? 112 : 96);
|
|
32
|
+
const resourceRva = buffer.readUInt32LE(dirOffset + 2 * 8);
|
|
33
|
+
if (!resourceRva) return [];
|
|
34
|
+
|
|
35
|
+
const sections = [];
|
|
36
|
+
for (let i = 0; i < numberOfSections; i += 1) {
|
|
37
|
+
const off = optional + sizeOfOptionalHeader + i * 40;
|
|
38
|
+
if (off + 40 > buffer.length) break;
|
|
39
|
+
sections.push({
|
|
40
|
+
virtualAddress: buffer.readUInt32LE(off + 12),
|
|
41
|
+
sizeOfRawData: buffer.readUInt32LE(off + 16),
|
|
42
|
+
pointerToRawData: buffer.readUInt32LE(off + 20),
|
|
43
|
+
virtualSize: buffer.readUInt32LE(off + 8),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const rvaToOffset = (rva) => {
|
|
48
|
+
for (const s of sections) {
|
|
49
|
+
const span = Math.max(s.sizeOfRawData, s.virtualSize);
|
|
50
|
+
if (rva >= s.virtualAddress && rva < s.virtualAddress + span) {
|
|
51
|
+
return s.pointerToRawData + (rva - s.virtualAddress);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return -1;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const rootOffset = rvaToOffset(resourceRva);
|
|
58
|
+
if (rootOffset < 0) return [];
|
|
59
|
+
|
|
60
|
+
const out = [];
|
|
61
|
+
const walk = (dirOffsetInFile, depth, typeId, nameId) => {
|
|
62
|
+
if (depth > 3 || dirOffsetInFile < 0 || dirOffsetInFile + 16 > buffer.length) return;
|
|
63
|
+
const named = buffer.readUInt16LE(dirOffsetInFile + 12);
|
|
64
|
+
const ids = buffer.readUInt16LE(dirOffsetInFile + 14);
|
|
65
|
+
for (let i = 0; i < named + ids; i += 1) {
|
|
66
|
+
const entry = dirOffsetInFile + 16 + i * 8;
|
|
67
|
+
if (entry + 8 > buffer.length) return;
|
|
68
|
+
const name = buffer.readUInt32LE(entry);
|
|
69
|
+
const offset = buffer.readUInt32LE(entry + 4);
|
|
70
|
+
const nextType = depth === 0 ? name : typeId;
|
|
71
|
+
const nextName = depth === 1 ? name : nameId;
|
|
72
|
+
if (offset & 0x80000000) {
|
|
73
|
+
walk(rootOffset + (offset & 0x7fffffff), depth + 1, nextType, nextName);
|
|
74
|
+
} else {
|
|
75
|
+
const dataEntry = rootOffset + offset;
|
|
76
|
+
if (dataEntry + 16 > buffer.length) continue;
|
|
77
|
+
const dataRva = buffer.readUInt32LE(dataEntry);
|
|
78
|
+
const dataSize = buffer.readUInt32LE(dataEntry + 4);
|
|
79
|
+
const dataOffset = rvaToOffset(dataRva);
|
|
80
|
+
if (dataOffset < 0 || dataOffset + dataSize > buffer.length) continue;
|
|
81
|
+
out.push({ type: nextType, name: nextName, offset: dataOffset, size: dataSize });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
walk(rootOffset, 0, null, null);
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Pulls windows-use.dll out of ArenaCode-x64.exe. Pure buffer work: testable anywhere. */
|
|
90
|
+
function extractDllFromExe(exePath) {
|
|
91
|
+
let buf;
|
|
92
|
+
try {
|
|
93
|
+
buf = fs.readFileSync(exePath);
|
|
94
|
+
} catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
for (const blob of readResourceBlobs(buf)) {
|
|
98
|
+
if (blob.size < 1024) continue;
|
|
99
|
+
const slice = buf.subarray(blob.offset, blob.offset + blob.size);
|
|
100
|
+
if (slice.readUInt16LE(0) !== 0x5a4d) continue;
|
|
101
|
+
if (!slice.includes(Buffer.from('wu_call_json'))) continue;
|
|
102
|
+
return Buffer.from(slice);
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ------------------------------------------------------------------ location
|
|
108
|
+
|
|
109
|
+
function cacheDir() {
|
|
110
|
+
const base = process.env.LOCALAPPDATA || path.join(os.homedir(), '.local', 'share');
|
|
111
|
+
return path.join(base, 'arenacode');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The DLL shipped inside the npm package: nothing else has to be installed. */
|
|
115
|
+
function packagedDll() {
|
|
116
|
+
return path.join(__dirname, '..', '..', 'vendor', 'windows-use', 'windows-use.dll');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function candidateExes() {
|
|
120
|
+
const out = [];
|
|
121
|
+
const push = (p) => {
|
|
122
|
+
if (p) out.push(p);
|
|
123
|
+
};
|
|
124
|
+
push(process.env.ARENACODE_EXE);
|
|
125
|
+
const home = os.homedir();
|
|
126
|
+
const appdata = process.env.APPDATA || path.join(home, 'AppData', 'Roaming');
|
|
127
|
+
const local = process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local');
|
|
128
|
+
const programFiles = process.env.ProgramFiles || 'C:\\Program Files';
|
|
129
|
+
const programFilesX86 = process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)';
|
|
130
|
+
for (const dir of [
|
|
131
|
+
path.join(local, 'ArenaCode'),
|
|
132
|
+
path.join(appdata, 'ArenaCode'),
|
|
133
|
+
path.join(programFiles, 'ArenaCode'),
|
|
134
|
+
path.join(programFilesX86, 'ArenaCode'),
|
|
135
|
+
path.join(home, 'ArenaCode'),
|
|
136
|
+
path.join(home, 'Desktop'),
|
|
137
|
+
path.join(home, 'Downloads'),
|
|
138
|
+
]) {
|
|
139
|
+
push(path.join(dir, 'ArenaCode-x64.exe'));
|
|
140
|
+
push(path.join(dir, 'ArenaCode.exe'));
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function candidateDlls() {
|
|
146
|
+
const out = [];
|
|
147
|
+
if (process.env.ARENACODE_WINDOWS_USE_DLL) out.push(process.env.ARENACODE_WINDOWS_USE_DLL);
|
|
148
|
+
out.push(packagedDll());
|
|
149
|
+
const home = os.homedir();
|
|
150
|
+
const local = process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local');
|
|
151
|
+
const appdata = process.env.APPDATA || path.join(home, 'AppData', 'Roaming');
|
|
152
|
+
out.push(path.join(cacheDir(), 'windows-use.dll'));
|
|
153
|
+
for (const dir of [path.join(local, 'ArenaCode'), path.join(appdata, 'ArenaCode'), path.dirname(process.execPath)]) {
|
|
154
|
+
out.push(path.join(dir, 'windows-use.dll'));
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
let dllPath = null;
|
|
160
|
+
let dllChecked = false;
|
|
161
|
+
|
|
162
|
+
function locate() {
|
|
163
|
+
if (dllChecked) return dllPath;
|
|
164
|
+
dllChecked = true;
|
|
165
|
+
for (const candidate of candidateDlls()) {
|
|
166
|
+
try {
|
|
167
|
+
if (fs.existsSync(candidate) && fs.statSync(candidate).size > 1024) {
|
|
168
|
+
dllPath = candidate;
|
|
169
|
+
return dllPath;
|
|
170
|
+
}
|
|
171
|
+
} catch {
|
|
172
|
+
/* ignore */
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// last resort: extract it from the desktop client, once, into the cache
|
|
176
|
+
for (const exe of candidateExes()) {
|
|
177
|
+
try {
|
|
178
|
+
if (!fs.existsSync(exe)) continue;
|
|
179
|
+
} catch {
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const blob = extractDllFromExe(exe);
|
|
183
|
+
if (!blob) continue;
|
|
184
|
+
for (const target of [path.join(cacheDir(), 'windows-use.dll'), path.join(os.tmpdir(), `arenacode-windows-use-${process.pid}.dll`)]) {
|
|
185
|
+
try {
|
|
186
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
187
|
+
fs.writeFileSync(target, blob);
|
|
188
|
+
dllPath = target;
|
|
189
|
+
return dllPath;
|
|
190
|
+
} catch {
|
|
191
|
+
/* try the next location */
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// --------------------------------------------------------------------- binding
|
|
199
|
+
|
|
200
|
+
let bindings = null;
|
|
201
|
+
let loadError = null;
|
|
202
|
+
|
|
203
|
+
function load() {
|
|
204
|
+
if (bindings) return bindings;
|
|
205
|
+
if (process.platform !== 'win32') {
|
|
206
|
+
loadError = 'windows-use.dll is only usable on Windows';
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
const file = locate();
|
|
210
|
+
if (!file) {
|
|
211
|
+
loadError = 'windows-use.dll not found (set ARENACODE_WINDOWS_USE_DLL, or reinstall the npm package)';
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
try {
|
|
215
|
+
const koffi = requireFfi();
|
|
216
|
+
const lib = koffi.load(file);
|
|
217
|
+
bindings = {
|
|
218
|
+
path: file,
|
|
219
|
+
koffi,
|
|
220
|
+
callJson: lib.func('const char* wu_call_json(const char* request)'),
|
|
221
|
+
free: lib.func('void wu_free(char* pointer)'),
|
|
222
|
+
version: lib.func('const char* wu_version()'),
|
|
223
|
+
};
|
|
224
|
+
return bindings;
|
|
225
|
+
} catch (err) {
|
|
226
|
+
loadError = `windows-use.dll could not be loaded: ${err.message}`;
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function version() {
|
|
232
|
+
const b = load();
|
|
233
|
+
if (!b) return null;
|
|
234
|
+
try {
|
|
235
|
+
return String(b.version());
|
|
236
|
+
} catch {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** Raw call: {"tool": "...", "args": {...}} in, {"ok":true,"result":…} out. */
|
|
242
|
+
function raw(request) {
|
|
243
|
+
const b = load();
|
|
244
|
+
if (!b) return { ok: false, error: loadError || 'windows-use.dll unavailable' };
|
|
245
|
+
let outPtr = null;
|
|
246
|
+
try {
|
|
247
|
+
outPtr = b.callJson(JSON.stringify(request));
|
|
248
|
+
if (!outPtr) return { ok: false, error: 'windows-use returned NULL' };
|
|
249
|
+
const text =
|
|
250
|
+
typeof outPtr === 'string' ? outPtr : String(b.koffi.decode(outPtr, 'char', 4 * 1024 * 1024)).split('\0')[0];
|
|
251
|
+
const parsed = JSON.parse(text);
|
|
252
|
+
return parsed && typeof parsed === 'object' ? parsed : { ok: false, error: 'unreadable answer' };
|
|
253
|
+
} catch (err) {
|
|
254
|
+
return { ok: false, error: `windows-use call failed: ${err.message}` };
|
|
255
|
+
} finally {
|
|
256
|
+
try {
|
|
257
|
+
if (outPtr && typeof outPtr !== 'string') b.free(outPtr);
|
|
258
|
+
} catch {
|
|
259
|
+
/* ignore */
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Tool names and argument keys, mined from the DLL's own symbols and string table
|
|
265
|
+
// (wu::api::{click_tool,desktop_state,…}), so nothing here is guessed at runtime.
|
|
266
|
+
const TOOLS = {
|
|
267
|
+
click: 'click_tool',
|
|
268
|
+
move: 'move_tool',
|
|
269
|
+
button: 'mouse_button_tool',
|
|
270
|
+
scroll: 'scroll_tool',
|
|
271
|
+
type: 'type_tool',
|
|
272
|
+
type_focus: 'type_focus_tool',
|
|
273
|
+
shortcut: 'shortcut_tool',
|
|
274
|
+
multi_select: 'multi_select_tool',
|
|
275
|
+
multi_edit: 'multi_edit_tool',
|
|
276
|
+
wait: 'wait_tool',
|
|
277
|
+
app: 'app_tool',
|
|
278
|
+
state: 'desktop_state',
|
|
279
|
+
apps: 'apps_list_tool',
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
function tool(kind, args = {}) {
|
|
283
|
+
const name = TOOLS[kind] || kind;
|
|
284
|
+
return raw({ tool: name, args });
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** The value inside {"ok":true,"result":…}, or {ok:false,error} unchanged. */
|
|
288
|
+
function result(answer) {
|
|
289
|
+
if (!answer || typeof answer !== 'object') return { ok: false, error: 'no answer' };
|
|
290
|
+
if (answer.ok === false) return { ok: false, error: answer.error || 'windows-use error' };
|
|
291
|
+
const value = 'result' in answer ? answer.result : answer;
|
|
292
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) return { ok: true, ...value };
|
|
293
|
+
return { ok: true, result: value };
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function pickWindow(list, name) {
|
|
297
|
+
const clean = (list || []).filter((w) => w && w.name);
|
|
298
|
+
if (!clean.length) return null;
|
|
299
|
+
const wanted = String(name || '').toLowerCase().trim();
|
|
300
|
+
if (!wanted) return clean[0];
|
|
301
|
+
return (
|
|
302
|
+
clean.find((w) => String(w.name).toLowerCase() === wanted) ||
|
|
303
|
+
clean.find((w) => String(w.name).toLowerCase().includes(wanted)) ||
|
|
304
|
+
clean.find((w) => wanted.includes(String(w.name).toLowerCase())) ||
|
|
305
|
+
null
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// ------------------------------------------------------------------ backend API
|
|
310
|
+
// Same shapes as src/automation.js, so the dispatcher can swap it in wholesale.
|
|
311
|
+
|
|
312
|
+
const backend = {
|
|
313
|
+
name: 'windows-use.dll',
|
|
314
|
+
|
|
315
|
+
available() {
|
|
316
|
+
const b = load();
|
|
317
|
+
if (!b) return { ok: false, backend: 'windows', error: loadError || 'windows-use.dll unavailable' };
|
|
318
|
+
return { ok: true, backend: 'windows', detail: `windows-use.dll ${version() || ''}`.trim(), dll: b.path };
|
|
319
|
+
},
|
|
320
|
+
|
|
321
|
+
screenSize() {
|
|
322
|
+
const r = backend.state();
|
|
323
|
+
if (r.ok && r.system && r.system.screen_size) {
|
|
324
|
+
return { ok: true, width: r.system.screen_size.width, height: r.system.screen_size.height };
|
|
325
|
+
}
|
|
326
|
+
return { ok: false, error: r.error || 'screen size unavailable' };
|
|
327
|
+
},
|
|
328
|
+
|
|
329
|
+
windows() {
|
|
330
|
+
const r = backend.state();
|
|
331
|
+
return r.ok ? { ok: true, windows: r.windows || [] } : r;
|
|
332
|
+
},
|
|
333
|
+
|
|
334
|
+
activeWindow() {
|
|
335
|
+
const r = backend.state();
|
|
336
|
+
if (!r.ok) return r;
|
|
337
|
+
const active = r.active_window || {};
|
|
338
|
+
return { ok: true, name: active.name || '', handle: active.handle ?? null };
|
|
339
|
+
},
|
|
340
|
+
|
|
341
|
+
findWindow(name) {
|
|
342
|
+
const list = backend.windows();
|
|
343
|
+
if (!list.ok) return list;
|
|
344
|
+
const hit = pickWindow(list.windows, name);
|
|
345
|
+
return hit ? { ok: true, window: hit } : { ok: false, error: `window not found: ${name}` };
|
|
346
|
+
},
|
|
347
|
+
|
|
348
|
+
state(handle = null) {
|
|
349
|
+
const args = { use_accessibility: true, use_vision: false, use_annotation: false };
|
|
350
|
+
if (handle) args.handle = handle;
|
|
351
|
+
const r = result(tool('state', args));
|
|
352
|
+
if (!r.ok) return r;
|
|
353
|
+
return {
|
|
354
|
+
ok: true,
|
|
355
|
+
windows: r.windows || [],
|
|
356
|
+
active_window: r.active_window || null,
|
|
357
|
+
interactive_elements: r.interactive_elements || [],
|
|
358
|
+
scrollable_elements: r.scrollable_elements || [],
|
|
359
|
+
interactive_elements_text: r.interactive_elements_text || '',
|
|
360
|
+
scrollable_elements_text: r.scrollable_elements_text || '',
|
|
361
|
+
tree_status: r.tree_status || '',
|
|
362
|
+
active_desktop: r.active_desktop ?? null,
|
|
363
|
+
all_desktops: r.all_desktops ?? null,
|
|
364
|
+
system: r.system || {},
|
|
365
|
+
metadata: { depth: r.depth, kind: r.kind, is_browser: r.is_browser },
|
|
366
|
+
};
|
|
367
|
+
},
|
|
368
|
+
|
|
369
|
+
moveMouse(x, y) {
|
|
370
|
+
const r = result(tool('move', { loc: [Math.round(x), Math.round(y)] }));
|
|
371
|
+
return r.ok ? { ok: true, x, y } : r;
|
|
372
|
+
},
|
|
373
|
+
|
|
374
|
+
click(x, y, { button = 'left', count = 1 } = {}) {
|
|
375
|
+
const clicks = Math.max(1, Math.min(2, Math.trunc(count) || 1));
|
|
376
|
+
const r = result(tool('click', { loc: [Math.round(x), Math.round(y)], button, clicks }));
|
|
377
|
+
return r.ok ? { ok: true, x, y, button, count } : r;
|
|
378
|
+
},
|
|
379
|
+
|
|
380
|
+
mouseButton(x, y, button, down) {
|
|
381
|
+
const r = result(tool('button', { loc: [Math.round(x), Math.round(y)], button, action: down ? 'down' : 'up' }));
|
|
382
|
+
return r.ok ? { ok: true, x, y, button } : r;
|
|
383
|
+
},
|
|
384
|
+
|
|
385
|
+
drag(x, y, x2, y2) {
|
|
386
|
+
const down = backend.mouseButton(x, y, 'left', true);
|
|
387
|
+
if (!down.ok) return down;
|
|
388
|
+
const moved = backend.moveMouse(x2, y2);
|
|
389
|
+
if (!moved.ok) return moved;
|
|
390
|
+
const up = backend.mouseButton(x2, y2, 'left', false);
|
|
391
|
+
return up.ok ? { ok: true, from: [x, y], to: [x2, y2] } : up;
|
|
392
|
+
},
|
|
393
|
+
|
|
394
|
+
scroll(amount, x = null, y = null) {
|
|
395
|
+
const times = Math.max(1, Math.abs(Math.trunc(amount) || 1));
|
|
396
|
+
const args = { direction: 'vertical', wheel_times: times };
|
|
397
|
+
if (x !== null && y !== null) args.loc = [Math.round(x), Math.round(y)];
|
|
398
|
+
const r = result(tool('scroll', args));
|
|
399
|
+
return r.ok ? { ok: true, amount } : r;
|
|
400
|
+
},
|
|
401
|
+
|
|
402
|
+
typeText(text, x = null, y = null) {
|
|
403
|
+
if (x !== null && y !== null) {
|
|
404
|
+
const clicked = backend.click(x, y);
|
|
405
|
+
if (!clicked.ok) return clicked;
|
|
406
|
+
}
|
|
407
|
+
const r = result(tool('type', { text: String(text) }));
|
|
408
|
+
return r.ok ? { ok: true, chars: String(text).length, typed: String(text).length } : r;
|
|
409
|
+
},
|
|
410
|
+
|
|
411
|
+
pressKeys(combo) {
|
|
412
|
+
const r = result(tool('shortcut', { shortcut: String(combo) }));
|
|
413
|
+
return r.ok ? { ok: true, keys: combo } : r;
|
|
414
|
+
},
|
|
415
|
+
|
|
416
|
+
focusWindow(name) {
|
|
417
|
+
const found = backend.findWindow(name);
|
|
418
|
+
if (!found.ok) return found;
|
|
419
|
+
const r = result(tool('app', { mode: 'switch', name: found.window.name }));
|
|
420
|
+
return r.ok ? { ok: true, name: found.window.name, handle: found.window.handle, matched: found.window.name } : r;
|
|
421
|
+
},
|
|
422
|
+
|
|
423
|
+
resizeWindow(name, x, y, width, height) {
|
|
424
|
+
const found = backend.findWindow(name);
|
|
425
|
+
if (!found.ok) return found;
|
|
426
|
+
const r = result(tool('app', { mode: 'resize', name: found.window.name, loc: [x, y], size: [width, height] }));
|
|
427
|
+
return r.ok ? { ok: true, name: found.window.name, x, y, width, height } : r;
|
|
428
|
+
},
|
|
429
|
+
|
|
430
|
+
launchApp(name) {
|
|
431
|
+
const r = result(tool('app', { mode: 'launch', name: String(name) }));
|
|
432
|
+
return r.ok ? { ok: true, app: name } : r;
|
|
433
|
+
},
|
|
434
|
+
|
|
435
|
+
listApps() {
|
|
436
|
+
const r = result(tool('apps', {}));
|
|
437
|
+
if (!r.ok) return r;
|
|
438
|
+
const apps = r.apps || (Array.isArray(r.result) ? r.result : []);
|
|
439
|
+
return { ok: true, apps };
|
|
440
|
+
},
|
|
441
|
+
|
|
442
|
+
screenshot(dest, { annotate = false } = {}) {
|
|
443
|
+
const r = result(tool('state', { use_accessibility: false, use_vision: true, use_annotation: Boolean(annotate) }));
|
|
444
|
+
if (!r.ok) return r;
|
|
445
|
+
const base64 = r.screenshot_png_base64 || '';
|
|
446
|
+
if (!base64) return { ok: false, error: 'no screenshot in windows-use answer' };
|
|
447
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
448
|
+
fs.writeFileSync(dest, Buffer.from(base64, 'base64'));
|
|
449
|
+
const size = (r.system && r.system.screen_size) || { width: 0, height: 0 };
|
|
450
|
+
return {
|
|
451
|
+
ok: true,
|
|
452
|
+
file: dest,
|
|
453
|
+
width: size.width,
|
|
454
|
+
height: size.height,
|
|
455
|
+
mime: 'image/png',
|
|
456
|
+
annotated: Boolean(annotate),
|
|
457
|
+
elements: (r.interactive_elements || []).length,
|
|
458
|
+
};
|
|
459
|
+
},
|
|
460
|
+
|
|
461
|
+
sysinfo() {
|
|
462
|
+
const r = result(tool('state', { use_accessibility: false, use_vision: false }));
|
|
463
|
+
const system = (r.ok && r.system) || {};
|
|
464
|
+
return {
|
|
465
|
+
ok: true,
|
|
466
|
+
system: {
|
|
467
|
+
windows_version: system.windows_version || os.release(),
|
|
468
|
+
os_version: system.os_version || os.release(),
|
|
469
|
+
screen_size: system.screen_size || { width: 0, height: 0 },
|
|
470
|
+
user_account_type: system.user_account_type || 'Local Account',
|
|
471
|
+
default_language: system.default_language || '',
|
|
472
|
+
dpi_scaling: system.dpi_scaling || 1,
|
|
473
|
+
},
|
|
474
|
+
};
|
|
475
|
+
},
|
|
476
|
+
};
|
|
477
|
+
|
|
478
|
+
module.exports = {
|
|
479
|
+
WU_EXPORTS,
|
|
480
|
+
TOOLS,
|
|
481
|
+
readResourceBlobs,
|
|
482
|
+
extractDllFromExe,
|
|
483
|
+
locate,
|
|
484
|
+
load,
|
|
485
|
+
version,
|
|
486
|
+
tool,
|
|
487
|
+
result,
|
|
488
|
+
raw,
|
|
489
|
+
backend,
|
|
490
|
+
get loadError() {
|
|
491
|
+
return loadError;
|
|
492
|
+
},
|
|
493
|
+
};
|
package/src/automation.js
CHANGED
|
@@ -990,4 +990,92 @@ const automation = {
|
|
|
990
990
|
},
|
|
991
991
|
};
|
|
992
992
|
|
|
993
|
+
|
|
994
|
+
// --------------------------------------------------------- native FFI dispatch --
|
|
995
|
+
// Windows → windows-use.dll (shipped in the package, same DLL as the desktop app)
|
|
996
|
+
// Linux → Xlib + XTest through koffi, no external tools at all
|
|
997
|
+
// macOS → unchanged (osascript + cliclick)
|
|
998
|
+
// The backends expose the same shapes as the object above, so installing one only
|
|
999
|
+
// replaces the methods; the hand-written engines stay as a fallback.
|
|
1000
|
+
|
|
1001
|
+
let nativeBackend = null;
|
|
1002
|
+
let nativeKind = null;
|
|
1003
|
+
|
|
1004
|
+
try {
|
|
1005
|
+
if (isLinux) {
|
|
1006
|
+
// Linux has no fallback any more: the FFI backend *is* the backend (the old one
|
|
1007
|
+
// shelled out to xdotool, which is exactly what we do not want).
|
|
1008
|
+
nativeBackend = require('./automation/linux').backend;
|
|
1009
|
+
nativeKind = 'x11';
|
|
1010
|
+
} else if (isWin) {
|
|
1011
|
+
// The DLL ships in the package; PowerShell stays only as a safety net for the case
|
|
1012
|
+
// where it cannot be loaded at all.
|
|
1013
|
+
const dll = require('./automation/win-dll');
|
|
1014
|
+
if (dll.load()) {
|
|
1015
|
+
nativeBackend = dll.backend;
|
|
1016
|
+
nativeKind = 'dll';
|
|
1017
|
+
}
|
|
1018
|
+
} else if (isMac) {
|
|
1019
|
+
const mac = require('./automation/mac');
|
|
1020
|
+
if (mac.available().ok) {
|
|
1021
|
+
nativeBackend = mac.backend;
|
|
1022
|
+
nativeKind = 'mac';
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
} catch (err) {
|
|
1026
|
+
nativeBackend = null;
|
|
1027
|
+
nativeKind = null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
if (nativeBackend) {
|
|
1031
|
+
const handleFor = (hit) => (nativeKind === 'dll' ? hit.name : hit.handle);
|
|
1032
|
+
const resolve = (name) => {
|
|
1033
|
+
const list = nativeBackend.windows();
|
|
1034
|
+
if (!list.ok) return null;
|
|
1035
|
+
return pickWindow(list.windows, name);
|
|
1036
|
+
};
|
|
1037
|
+
|
|
1038
|
+
Object.assign(automation, {
|
|
1039
|
+
native: {
|
|
1040
|
+
kind: nativeKind,
|
|
1041
|
+
name: nativeBackend.name,
|
|
1042
|
+
dll: nativeKind === 'dll' ? require('./automation/win-dll').locate() : null,
|
|
1043
|
+
},
|
|
1044
|
+
available: () => nativeBackend.available(),
|
|
1045
|
+
screenSize: () => nativeBackend.screenSize(),
|
|
1046
|
+
windows: () => nativeBackend.windows(),
|
|
1047
|
+
activeWindow: () => nativeBackend.activeWindow(),
|
|
1048
|
+
findWindow(name) {
|
|
1049
|
+
const list = nativeBackend.windows();
|
|
1050
|
+
if (!list.ok) return list;
|
|
1051
|
+
const hit = pickWindow(list.windows, name);
|
|
1052
|
+
return hit ? { ok: true, window: hit } : fail(`window not found: ${name}`);
|
|
1053
|
+
},
|
|
1054
|
+
state: (handle = null) => nativeBackend.state(handle),
|
|
1055
|
+
moveMouse: (x, y) => nativeBackend.moveMouse(x, y),
|
|
1056
|
+
click: (x, y, opts = {}) => nativeBackend.click(x, y, opts),
|
|
1057
|
+
mouseButton: (x, y, button, down) => nativeBackend.mouseButton(x, y, button, down),
|
|
1058
|
+
drag: (x, y, x2, y2) => nativeBackend.drag(x, y, x2, y2),
|
|
1059
|
+
scroll: (amount, x = null, y = null) => nativeBackend.scroll(amount, x, y),
|
|
1060
|
+
typeText: (text, x = null, y = null) => nativeBackend.typeText(text, x, y),
|
|
1061
|
+
pressKeys: (combo) => nativeBackend.pressKeys(combo),
|
|
1062
|
+
screenshot: (dest, opts = {}) => nativeBackend.screenshot(dest, opts),
|
|
1063
|
+
listApps: () => nativeBackend.listApps(),
|
|
1064
|
+
sysinfo: () => nativeBackend.sysinfo(),
|
|
1065
|
+
launchApp: (name) => nativeBackend.launchApp(name),
|
|
1066
|
+
focusWindow(name) {
|
|
1067
|
+
const hit = resolve(name);
|
|
1068
|
+
if (!hit) return fail(`window not found: ${name}`);
|
|
1069
|
+
const answer = nativeBackend.focusWindow(handleFor(hit));
|
|
1070
|
+
return answer.ok ? { ...answer, name: hit.name, handle: hit.handle, matched: hit.name } : answer;
|
|
1071
|
+
},
|
|
1072
|
+
resizeWindow(name, x, y, width, height) {
|
|
1073
|
+
const hit = resolve(name);
|
|
1074
|
+
if (!hit) return fail(`window not found: ${name}`);
|
|
1075
|
+
const answer = nativeBackend.resizeWindow(handleFor(hit), x, y, width, height);
|
|
1076
|
+
return answer.ok ? { ...answer, name: hit.name, x, y, width, height } : answer;
|
|
1077
|
+
},
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
|
|
993
1081
|
module.exports = { ...automation, keyboardSequence, fuzzyScore, pickWindow };
|
package/src/cli.js
CHANGED
|
@@ -103,6 +103,7 @@ function context(flags) {
|
|
|
103
103
|
json: Boolean(flags.json),
|
|
104
104
|
confirmWrites: Boolean(flags['confirm-writes']),
|
|
105
105
|
approval: normaliseApproval(flags.proceed ? 'proceed' : flags.ask ? 'ask' : cfg.approval),
|
|
106
|
+
shell: String(flags.shell || process.env.ARENACODE_SHELL || cfg.shell || '').trim(),
|
|
106
107
|
idleTimeout: Number(flags['idle-timeout'] || cfg.idle_timeout || 180),
|
|
107
108
|
bridge: new Bridge(url, key, device),
|
|
108
109
|
};
|
|
@@ -194,7 +195,7 @@ function cmdConfig(positional, flags) {
|
|
|
194
195
|
const name = key;
|
|
195
196
|
const val = rest.join(' ');
|
|
196
197
|
if (!name || val === '') {
|
|
197
|
-
write(`usage: ${SELF} config set <url|key|lang|mode|approval|pc_access> <value>`);
|
|
198
|
+
write(`usage: ${SELF} config set <url|key|lang|mode|approval|pc_access|shell> <value>`);
|
|
198
199
|
return 2;
|
|
199
200
|
}
|
|
200
201
|
if (name === 'approval' && !['ask', 'proceed'].includes(String(val))) {
|
|
@@ -337,6 +338,7 @@ async function runInstruction(ctx, text, { attachments = [] } = {}) {
|
|
|
337
338
|
const result = await followTask(ctx.bridge, taskId, {
|
|
338
339
|
folder: ctx.folder,
|
|
339
340
|
lang: ctx.lang,
|
|
341
|
+
shell: ctx.shell,
|
|
340
342
|
yes: ctx.yes,
|
|
341
343
|
approval: ctx.approval,
|
|
342
344
|
confirmWrites: ctx.confirmWrites,
|
|
@@ -400,6 +402,7 @@ async function cmdUi(flags) {
|
|
|
400
402
|
yes: ctx.yes,
|
|
401
403
|
approval: ctx.approval,
|
|
402
404
|
confirmWrites: ctx.confirmWrites,
|
|
405
|
+
shell: ctx.shell,
|
|
403
406
|
});
|
|
404
407
|
await app.start();
|
|
405
408
|
return 0;
|
|
@@ -688,7 +691,7 @@ async function cmdHandshake(flags) {
|
|
|
688
691
|
const USAGE = {
|
|
689
692
|
init: [['init', 'configure bridge URL + API key, then test the connection']],
|
|
690
693
|
config: [['config [show|path]', 'read the saved configuration'],
|
|
691
|
-
['config set <key> <value>', 'url · key · lang · mode · pc_access']],
|
|
694
|
+
['config set <key> <value>', 'url · key · lang · mode · pc_access · shell']],
|
|
692
695
|
ping: [['ping', 'bridge status and API key check']],
|
|
693
696
|
doctor: [['doctor', 'local checks: node, platform, folder, git, bridge']],
|
|
694
697
|
ask: [['ask "instruction"', 'run one task and stream the timeline']],
|