beast-agent 2.44.8 → 2.44.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/computeruse.js +200 -837
- package/src/agent/engine.js +8 -25
- package/src/agent/watext.js +1 -1
- package/src/main.js +1 -22
- package/src/preload.js +0 -1
- package/src/renderer/renderer.js +0 -5
package/package.json
CHANGED
package/src/agent/computeruse.js
CHANGED
|
@@ -1,837 +1,200 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/* Beast Computer Use
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
};
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const list = await rpc('tools/list', {}, 15000);
|
|
202
|
-
S.tools = new Set(((list && list.tools) || []).map((t) => t.name));
|
|
203
|
-
S.booted = true;
|
|
204
|
-
finish(true);
|
|
205
|
-
} catch {
|
|
206
|
-
teardown();
|
|
207
|
-
finish(false);
|
|
208
|
-
}
|
|
209
|
-
})();
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
let starting = null;
|
|
214
|
-
async function ensure() {
|
|
215
|
-
if (S.proc && S.booted) return true;
|
|
216
|
-
if (Date.now() < S.downUntil) {
|
|
217
|
-
maybeAutoInstall(); /* cooldown'da da kurulum eksikse başlatılsın */
|
|
218
|
-
return false;
|
|
219
|
-
}
|
|
220
|
-
if (!starting) {
|
|
221
|
-
starting = boot().finally(() => { starting = null; });
|
|
222
|
-
}
|
|
223
|
-
return starting;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
function writeLine(obj) {
|
|
227
|
-
if (!S.proc || !S.proc.stdin.writable) return;
|
|
228
|
-
try { S.proc.stdin.write(JSON.stringify(obj) + '\n'); } catch {}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
function rpc(method, params, timeoutMs) {
|
|
232
|
-
return new Promise((resolve, reject) => {
|
|
233
|
-
const id = S.nextId++;
|
|
234
|
-
const timer = setTimeout(() => {
|
|
235
|
-
S.pending.delete(id);
|
|
236
|
-
reject(new Error('cua-driver zaman aşımı: ' + method));
|
|
237
|
-
}, timeoutMs || 30000);
|
|
238
|
-
S.pending.set(id, {
|
|
239
|
-
resolve: (v) => { clearTimeout(timer); resolve(v); },
|
|
240
|
-
reject: (e) => { clearTimeout(timer); reject(e); },
|
|
241
|
-
});
|
|
242
|
-
writeLine({ jsonrpc: '2.0', id, method, params: params || {} });
|
|
243
|
-
});
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/* MCP CallTool zarfını { isError, text, structured, image } biçimine indirger */
|
|
247
|
-
function readResult(res) {
|
|
248
|
-
let text = '';
|
|
249
|
-
let image = null;
|
|
250
|
-
for (const c of Array.isArray(res && res.content) ? res.content : []) {
|
|
251
|
-
if (c && c.type === 'text') text += (text ? '\n' : '') + String(c.text || '');
|
|
252
|
-
else if (c && c.type === 'image') image = 'data:' + (c.mimeType || 'image/png') + ';base64,' + c.data;
|
|
253
|
-
}
|
|
254
|
-
text = text.replace(/^\u2705+\s*/, '').trim();
|
|
255
|
-
return { isError: !!res.isError, text, structured: (res && res.structuredContent) || null, image };
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
function hasTool(name) {
|
|
259
|
-
return !S.tools || S.tools.has(name); /* tools/list boş döndüyse geç */
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
async function call(tool, args, timeoutMs) {
|
|
263
|
-
if (!(await ensure())) throw new Error('cua-driver yok');
|
|
264
|
-
if (!hasTool(tool)) throw new Error('cua-driver bu aksiyonu desteklemiyor: ' + tool);
|
|
265
|
-
const res = await rpc('tools/call', { name: tool, arguments: args || {} }, timeoutMs || CALL_TIMEOUT);
|
|
266
|
-
return readResult(res);
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
/* structuredContent + metin fallback'ından satır dizisi çıkarır */
|
|
270
|
-
function rowsOf(r) {
|
|
271
|
-
const s = r && r.structured;
|
|
272
|
-
if (Array.isArray(s)) return s;
|
|
273
|
-
if (s && Array.isArray(s.windows)) return s.windows;
|
|
274
|
-
if (s && Array.isArray(s.elements)) return s.elements;
|
|
275
|
-
if (s && Array.isArray(s.apps)) return s.apps;
|
|
276
|
-
if (r && r.text) {
|
|
277
|
-
try {
|
|
278
|
-
const m = r.text.match(/\[[\s\S]*\]/);
|
|
279
|
-
if (m) return JSON.parse(m[0]);
|
|
280
|
-
} catch {}
|
|
281
|
-
}
|
|
282
|
-
return [];
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/* ---------- hedef seçimi ---------- */
|
|
286
|
-
|
|
287
|
-
async function findTarget(app) {
|
|
288
|
-
const r = await call('list_windows', { on_screen_only: true });
|
|
289
|
-
let cands = rowsOf(r).filter((w) => w && w.pid != null && w.window_id != null);
|
|
290
|
-
if (app) {
|
|
291
|
-
const q = String(app).toLowerCase();
|
|
292
|
-
cands = cands.filter((w) =>
|
|
293
|
-
String(w.app_name || '').toLowerCase().includes(q) ||
|
|
294
|
-
String(w.title || '').toLowerCase().includes(q));
|
|
295
|
-
}
|
|
296
|
-
if (!cands.length) return null;
|
|
297
|
-
cands.sort((a, b) => (Number(b.z_index) || 0) - (Number(a.z_index) || 0));
|
|
298
|
-
return {
|
|
299
|
-
pid: Number(cands[0].pid),
|
|
300
|
-
window_id: Number(cands[0].window_id),
|
|
301
|
-
app_name: String(cands[0].app_name || ''),
|
|
302
|
-
title: String(cands[0].title || ''),
|
|
303
|
-
};
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
/* capture → snapshot cache + Hermes tarzı numaralı index (+ opsiyonel görsel) */
|
|
307
|
-
async function doCapture(args = {}) {
|
|
308
|
-
const t = args.app || !S.target ? await findTarget(args.app) : S.target;
|
|
309
|
-
if (!t) {
|
|
310
|
-
return { ok: false, error: 'hedef pencere bulunamadı' + (args.app ? ': ' + args.app : '') };
|
|
311
|
-
}
|
|
312
|
-
S.target = t;
|
|
313
|
-
const mode = args.mode === 'vision' || args.mode === 'ax' ? args.mode : 'som';
|
|
314
|
-
const wargs = { pid: t.pid, window_id: t.window_id };
|
|
315
|
-
if (mode === 'vision') wargs.include_accessibility_tree = false;
|
|
316
|
-
if (mode === 'ax') wargs.include_screenshot = false;
|
|
317
|
-
const r = await call('get_window_state', wargs);
|
|
318
|
-
if (r.isError) return { ok: false, error: (r.text || 'capture başarısız').slice(0, 300) };
|
|
319
|
-
const s = r.structured || {};
|
|
320
|
-
const elements = Array.isArray(s.elements) ? s.elements : [];
|
|
321
|
-
S.snapshot = {
|
|
322
|
-
id: String(s.snapshot_id || s.snapshotId || ''),
|
|
323
|
-
pid: t.pid,
|
|
324
|
-
window_id: t.window_id,
|
|
325
|
-
map: new Map(),
|
|
326
|
-
at: Date.now(),
|
|
327
|
-
};
|
|
328
|
-
for (const el of elements) {
|
|
329
|
-
if (el && el.element_index != null) S.snapshot.map.set(Number(el.element_index), el);
|
|
330
|
-
}
|
|
331
|
-
const out = {
|
|
332
|
-
ok: true,
|
|
333
|
-
op: 'capture',
|
|
334
|
-
app: t.app_name,
|
|
335
|
-
window: { pid: t.pid, window_id: t.window_id, title: t.title },
|
|
336
|
-
elements: elements.length,
|
|
337
|
-
index: elements.slice(0, 200).map((el) => {
|
|
338
|
-
const f = el.frame || {};
|
|
339
|
-
const val = el.value != null && String(el.value).length ? " = '" + String(el.value).slice(0, 40) + "'" : '';
|
|
340
|
-
return `#${el.element_index} ${el.role || 'Element'} '${String(el.label || '').slice(0, 60)}'${val} @ (${f.x ?? 0}, ${f.y ?? 0}, ${f.w ?? 0}, ${f.h ?? 0})`;
|
|
341
|
-
}).join('\n'),
|
|
342
|
-
note: 'element index ile hedefle: {op:"click", element:N}; element bayatlarsa yeniden capture',
|
|
343
|
-
};
|
|
344
|
-
if (mode !== 'ax' && r.image) out.__injectImage = r.image;
|
|
345
|
-
return out;
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
/* snapshot element çözümü — bayat/bilinmeyen index açık hata verir */
|
|
349
|
-
function resolveEl(args) {
|
|
350
|
-
const idx = Number(args && args.element);
|
|
351
|
-
if (!idx) return {};
|
|
352
|
-
const snap = S.snapshot;
|
|
353
|
-
if (!snap || !snap.map.has(idx)) {
|
|
354
|
-
return { error: `element #${idx} bulunamadı/bayatladı — önce capture`, stale: true };
|
|
355
|
-
}
|
|
356
|
-
return { el: snap.map.get(idx) };
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
/* aksiyon argümanlarını cua-driver hedef biçimine çevirir */
|
|
360
|
-
function address(args, t) {
|
|
361
|
-
const r = resolveEl(args);
|
|
362
|
-
if (r.error) return r;
|
|
363
|
-
const out = {};
|
|
364
|
-
if (r.el) {
|
|
365
|
-
out.pid = t.pid;
|
|
366
|
-
const tk = r.el.element_token || r.el.token;
|
|
367
|
-
if (tk) out.element_token = tk;
|
|
368
|
-
else {
|
|
369
|
-
out.element_index = Number(r.el.element_index);
|
|
370
|
-
out.snapshot_id = S.snapshot.id;
|
|
371
|
-
out.window_id = S.snapshot.window_id;
|
|
372
|
-
}
|
|
373
|
-
return { out };
|
|
374
|
-
}
|
|
375
|
-
if (args.x != null || args.y != null) {
|
|
376
|
-
out.pid = t.pid;
|
|
377
|
-
out.x = Number(args.x) || 0;
|
|
378
|
-
out.y = Number(args.y) || 0;
|
|
379
|
-
return { out };
|
|
380
|
-
}
|
|
381
|
-
/* hedef yok: yalnız uygulama hedefli (focused element) */
|
|
382
|
-
if (args.app) out.pid = t.pid;
|
|
383
|
-
return { out };
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
function verdict(r) {
|
|
387
|
-
if (r.isError) return { ok: false, error: (r.text || 'cua-driver hatası').slice(0, 300) };
|
|
388
|
-
const s = r.structured || {};
|
|
389
|
-
const out = { ok: true, effect: s.effect, verified: s.verified, path: s.path, code: s.code };
|
|
390
|
-
if (s.escalation) out.escalation = s.escalation;
|
|
391
|
-
if (r.text) out.note = r.text.slice(0, 300);
|
|
392
|
-
return out;
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
/* ---------- güvenlik (Hermes SKILL.md sert kuralları) ---------- */
|
|
396
|
-
|
|
397
|
-
const BLOCKED_TYPE_PATTERNS = [
|
|
398
|
-
/\b(?:curl|wget|irm|iwr|invoke-webrequest)\b[^|\n]*\|\s*(?:&\s*)?(?:bash|sh|zsh|powershell|pwsh|iex|invoke-expression)\b/i,
|
|
399
|
-
/\bsudo\s+rm\s+-rf\b/i,
|
|
400
|
-
/\brm\s+-rf\s+(?:\/|~)/,
|
|
401
|
-
/:\(\)\s*\{[^}]*\}\s*;\s*:/,
|
|
402
|
-
];
|
|
403
|
-
const BLOCKED_KEYS = [/^win\+l$/, /^ctrl\+alt\+del(?:ete)?$/, /^alt\+ctrl\+del(?:ete)?$/];
|
|
404
|
-
|
|
405
|
-
const KEY_ALIASES = {
|
|
406
|
-
enter: 'return', esc: 'escape', del: 'delete', pgup: 'pageup', pgdn: 'pagedown',
|
|
407
|
-
win: 'cmd', option: 'alt', control: 'ctrl',
|
|
408
|
-
};
|
|
409
|
-
const KEY_MODS = new Set(['ctrl', 'alt', 'shift', 'cmd', 'fn']);
|
|
410
|
-
|
|
411
|
-
function guard(op, args) {
|
|
412
|
-
if (op === 'type') {
|
|
413
|
-
const text = String(args.text ?? '');
|
|
414
|
-
if (!text.trim()) return { ok: false, error: 'boş metin' };
|
|
415
|
-
for (const re of BLOCKED_TYPE_PATTERNS) {
|
|
416
|
-
if (re.test(text)) return { ok: false, error: 'şüpheli shell pattern bloklu', blocked: true };
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
if (op === 'key') {
|
|
420
|
-
const combo = String(args.keys || args.combo || '').trim().toLowerCase();
|
|
421
|
-
if (!combo) return { ok: false, error: 'tuş gerekli' };
|
|
422
|
-
const norm = combo.replace(/\s+/g, '');
|
|
423
|
-
for (const re of BLOCKED_KEYS) {
|
|
424
|
-
if (re.test(norm)) return { ok: false, error: 'kilit/logout kombinasyonu bloklu: ' + combo, blocked: true };
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
return null;
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
/* ---------- cua-driver aksiyonları ---------- */
|
|
431
|
-
|
|
432
|
-
async function targetFor(args) {
|
|
433
|
-
if (!args || (!args.app && S.target)) return S.target || (await findTarget(null));
|
|
434
|
-
return findTarget(args && args.app);
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
function keyParts(combo) {
|
|
438
|
-
const parts = String(combo).toLowerCase().split('+').map((p) => p.trim()).filter(Boolean);
|
|
439
|
-
const mods = [];
|
|
440
|
-
let key = '';
|
|
441
|
-
for (const p of parts) {
|
|
442
|
-
const k = KEY_ALIASES[p] || p;
|
|
443
|
-
if (KEY_MODS.has(k)) mods.push(k);
|
|
444
|
-
else key = k;
|
|
445
|
-
}
|
|
446
|
-
return { mods, key };
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
async function driverAct(op, args) {
|
|
450
|
-
const t = await targetFor(args);
|
|
451
|
-
if (!t && op !== 'list_apps') {
|
|
452
|
-
return { ok: false, error: 'hedef pencere bulunamadı' + (args.app ? ': ' + args.app : '') };
|
|
453
|
-
}
|
|
454
|
-
switch (op) {
|
|
455
|
-
case 'click':
|
|
456
|
-
case 'dblclick':
|
|
457
|
-
case 'rightclick': {
|
|
458
|
-
const a = address(args, t);
|
|
459
|
-
if (a.error) return a;
|
|
460
|
-
if (!a.out.element_token && a.out.element_index == null && a.out.x == null) {
|
|
461
|
-
return { ok: false, error: 'element veya x/y gerekli' };
|
|
462
|
-
}
|
|
463
|
-
const p = a.out;
|
|
464
|
-
if (op === 'click') {
|
|
465
|
-
p.button = ['left', 'right', 'middle'].includes(args.button) ? args.button : 'left';
|
|
466
|
-
return verdict(await call('click', p));
|
|
467
|
-
}
|
|
468
|
-
if (op === 'dblclick') return verdict(await call('double_click', p));
|
|
469
|
-
return verdict(await call('right_click', p));
|
|
470
|
-
}
|
|
471
|
-
case 'type': {
|
|
472
|
-
const text = String(args.text ?? '').slice(0, 2000);
|
|
473
|
-
const a = address(args, t);
|
|
474
|
-
if (a.error) return a;
|
|
475
|
-
return verdict(await call('type_text', { text, ...a.out }));
|
|
476
|
-
}
|
|
477
|
-
case 'key': {
|
|
478
|
-
const combo = String(args.keys || args.combo || '').trim().toLowerCase();
|
|
479
|
-
const { mods, key } = keyParts(combo);
|
|
480
|
-
if (!key && !mods.length) return { ok: false, error: 'tuş gerekli' };
|
|
481
|
-
const a = address(args, t);
|
|
482
|
-
if (a.error) return a;
|
|
483
|
-
if (mods.length) {
|
|
484
|
-
return verdict(await call('hotkey', { keys: [...mods, key || ''], ...a.out }));
|
|
485
|
-
}
|
|
486
|
-
return verdict(await call('press_key', { key: key || 'return', ...a.out }));
|
|
487
|
-
}
|
|
488
|
-
case 'scroll': {
|
|
489
|
-
const direction = ['up', 'down', 'left', 'right'].includes(args.direction) ? args.direction : null;
|
|
490
|
-
if (!direction) return { ok: false, error: 'direction gerekli (up/down/left/right)' };
|
|
491
|
-
const amount = Math.max(1, Math.min(50, Math.round(Number(args.amount) || 3)));
|
|
492
|
-
const a = address(args, t);
|
|
493
|
-
if (a.error) return a;
|
|
494
|
-
return verdict(await call('scroll', { direction, amount, ...a.out }));
|
|
495
|
-
}
|
|
496
|
-
case 'drag': {
|
|
497
|
-
const pt = async (elIdx, ax, ay) => {
|
|
498
|
-
const r = elIdx ? resolveEl({ element: elIdx }) : {};
|
|
499
|
-
if (r.error) return r;
|
|
500
|
-
if (r.el && r.el.frame) {
|
|
501
|
-
return { x: r.el.frame.x + r.el.frame.w / 2, y: r.el.frame.y + r.el.frame.h / 2 };
|
|
502
|
-
}
|
|
503
|
-
if (ax != null && ay != null) return { x: Number(ax), y: Number(ay) };
|
|
504
|
-
return { error: 'drag: from/to (element veya x/y) gerekli' };
|
|
505
|
-
};
|
|
506
|
-
const from = await pt(args.from_element, args.from_x, args.from_y);
|
|
507
|
-
if (from.error) return from;
|
|
508
|
-
const to = await pt(args.to_element, args.to_x, args.to_y);
|
|
509
|
-
if (to.error) return to;
|
|
510
|
-
const p = { from_x: from.x, from_y: from.y, to_x: to.x, to_y: to.y };
|
|
511
|
-
if (args.duration_ms) p.duration_ms = Math.max(0, Math.min(10000, Math.round(Number(args.duration_ms))));
|
|
512
|
-
if (args.steps) p.steps = Math.max(1, Math.min(200, Math.round(Number(args.steps))));
|
|
513
|
-
return verdict(await call('drag', p));
|
|
514
|
-
}
|
|
515
|
-
case 'list_apps': {
|
|
516
|
-
const r = await call('list_apps');
|
|
517
|
-
if (r.isError) return { ok: false, error: (r.text || 'list_apps başarısız').slice(0, 300) };
|
|
518
|
-
const rows = rowsOf(r).slice(0, 40).map((a) => ({
|
|
519
|
-
name: a.name || a.app_name || a.bundle_id || String(a).slice(0, 40),
|
|
520
|
-
pid: a.pid || null,
|
|
521
|
-
running: a.running !== false,
|
|
522
|
-
}));
|
|
523
|
-
return { ok: true, op: 'list_apps', apps: rows, note: 'hedeflemek için op argümanlarında app=<ad> kullan' };
|
|
524
|
-
}
|
|
525
|
-
case 'focus_app': {
|
|
526
|
-
S.target = t;
|
|
527
|
-
if (args.raise_window) {
|
|
528
|
-
const r = await call('bring_to_front', { pid: t.pid });
|
|
529
|
-
if (r.isError) return { ok: false, error: (r.text || 'focus başarısız').slice(0, 300) };
|
|
530
|
-
}
|
|
531
|
-
return {
|
|
532
|
-
ok: true,
|
|
533
|
-
op: 'focus_app',
|
|
534
|
-
app: t.app_name,
|
|
535
|
-
window: { pid: t.pid, window_id: t.window_id, title: t.title },
|
|
536
|
-
note: 'hedef seçildi — girdiler artık bu uygulamaya yönelir (raise_window olmadan odak çalınmaz)',
|
|
537
|
-
};
|
|
538
|
-
}
|
|
539
|
-
default:
|
|
540
|
-
return null; /* move vb. → legacy */
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
/* capture_after: aksiyon sonrası aynı hedefte doğrulama görüntüsü */
|
|
545
|
-
async function attachCapture(out) {
|
|
546
|
-
try {
|
|
547
|
-
const c = await doCapture({});
|
|
548
|
-
if (c && c.__injectImage) {
|
|
549
|
-
out.__injectImage = c.__injectImage;
|
|
550
|
-
delete c.__injectImage;
|
|
551
|
-
}
|
|
552
|
-
out.capture = { ok: !!c.ok, elements: c.elements, index: c.index || undefined, error: c.error };
|
|
553
|
-
} catch (e) {
|
|
554
|
-
out.capture = { ok: false, error: String((e && e.message) || e) };
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
/* ---------- legacy PowerShell backend (fallback) ---------- */
|
|
559
|
-
|
|
560
|
-
let lastActionAt = 0;
|
|
561
|
-
function pace() {
|
|
562
|
-
const wait = 120 - (Date.now() - lastActionAt);
|
|
563
|
-
lastActionAt = Date.now();
|
|
564
|
-
if (wait > 0) return new Promise((r) => setTimeout(r, wait));
|
|
565
|
-
return Promise.resolve();
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
function sleep(ms) {
|
|
569
|
-
return new Promise((r) => setTimeout(r, ms));
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
function ps(script) {
|
|
573
|
-
return new Promise((resolve, reject) => {
|
|
574
|
-
execFile(
|
|
575
|
-
'powershell.exe',
|
|
576
|
-
['-NoProfile', '-NonInteractive', '-Command', script],
|
|
577
|
-
{ timeout: 20000, windowsHide: true },
|
|
578
|
-
(err, stdout) => (err ? reject(err) : resolve(String(stdout || '').trim()))
|
|
579
|
-
);
|
|
580
|
-
});
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
function clamp(n, lo, hi) {
|
|
584
|
-
return Math.max(lo, Math.min(hi, n));
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
async function legacyScreenSize() {
|
|
588
|
-
const out = await ps(
|
|
589
|
-
'Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width.ToString() + "x" + [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Height'
|
|
590
|
-
);
|
|
591
|
-
const m = out.match(/(\d+)x(\d+)/);
|
|
592
|
-
if (!m) throw new Error('çözünürlük alınamadı');
|
|
593
|
-
return { w: Number(m[1]), h: Number(m[2]) };
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
const MOUSE_MOVE_TPL = `
|
|
597
|
-
Add-Type @"
|
|
598
|
-
using System;
|
|
599
|
-
using System.Runtime.InteropServices;
|
|
600
|
-
public class BM {
|
|
601
|
-
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
|
602
|
-
}
|
|
603
|
-
"@
|
|
604
|
-
[BM]::SetCursorPos(%%X%%, %%Y%%) | Out-Null
|
|
605
|
-
`;
|
|
606
|
-
|
|
607
|
-
const WHEEL_TPL = `
|
|
608
|
-
Add-Type @"
|
|
609
|
-
using System;
|
|
610
|
-
using System.Runtime.InteropServices;
|
|
611
|
-
public class BW {
|
|
612
|
-
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
|
|
613
|
-
}
|
|
614
|
-
"@
|
|
615
|
-
[BW]::mouse_event(0x0800, 0, 0, %%DELTA%%, [UIntPtr]::Zero)
|
|
616
|
-
`;
|
|
617
|
-
|
|
618
|
-
/* Unicode güvenli yazım: pano + Ctrl+V en sağlamı (tek seferlik, panoyu geri yüklemez) */
|
|
619
|
-
function buildTyperScript(text) {
|
|
620
|
-
const b64 = Buffer.from(text, 'utf16le').toString('base64');
|
|
621
|
-
return `
|
|
622
|
-
$t = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${b64}"))
|
|
623
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
624
|
-
[System.Windows.Forms.Clipboard]::SetText($t)
|
|
625
|
-
Start-Sleep -Milliseconds 80
|
|
626
|
-
[System.Windows.Forms.SendKeys]::SendWait("^v")
|
|
627
|
-
`;
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
/* Tuş kombosu: enter/tab/esc/f1-12, ctrl/alt/shift+x, win vb. */
|
|
631
|
-
function buildKeyScript(combo) {
|
|
632
|
-
const map = {
|
|
633
|
-
enter: '{ENTER}', tab: '{TAB}', esc: '{ESC}', escape: '{ESC}',
|
|
634
|
-
backspace: '{BACKSPACE}', delete: '{DELETE}', del: '{DELETE}',
|
|
635
|
-
home: '{HOME}', end: '{END}', pgup: '{PGUP}', pgdn: '{PGDN}',
|
|
636
|
-
up: '{UP}', down: '{DOWN}', left: '{LEFT}', right: '{RIGHT}',
|
|
637
|
-
space: ' ', win: '^({ESC})',
|
|
638
|
-
};
|
|
639
|
-
let out = '';
|
|
640
|
-
const parts = String(combo).toLowerCase().split('+').map((p) => p.trim());
|
|
641
|
-
let mods = '';
|
|
642
|
-
for (const p of parts) {
|
|
643
|
-
if (p === 'ctrl') mods += '^';
|
|
644
|
-
else if (p === 'alt') mods += '%';
|
|
645
|
-
else if (p === 'shift') mods += '+';
|
|
646
|
-
else if (p === 'win') out += '';
|
|
647
|
-
else if (/^f\d{1,2}$/.test(p)) out += '{' + p.toUpperCase() + '}';
|
|
648
|
-
else if (map[p]) out += map[p];
|
|
649
|
-
else out += p.slice(0, 1).toUpperCase();
|
|
650
|
-
}
|
|
651
|
-
const seq = mods && out ? mods + '(' + out + ')' : mods + out;
|
|
652
|
-
const b64 = Buffer.from(seq, 'utf16le').toString('base64');
|
|
653
|
-
return `
|
|
654
|
-
$s = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${b64}"))
|
|
655
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
656
|
-
[System.Windows.Forms.SendKeys]::SendWait($s)
|
|
657
|
-
`;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
async function mouseMove(x, y) {
|
|
661
|
-
await ps(MOUSE_MOVE_TPL.replace(/%%X%%/g, String(x)).replace(/%%Y%%/g, String(y)));
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
async function clickSimple(button) {
|
|
665
|
-
const down = button === 'right' ? 0x0008 : 0x0002;
|
|
666
|
-
const up = button === 'right' ? 0x0010 : 0x0004;
|
|
667
|
-
await ps(`
|
|
668
|
-
Add-Type @"
|
|
669
|
-
using System;
|
|
670
|
-
using System.Runtime.InteropServices;
|
|
671
|
-
public class BC2 {
|
|
672
|
-
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
|
|
673
|
-
}
|
|
674
|
-
"@
|
|
675
|
-
[BC2]::mouse_event(${down}, 0, 0, 0, [UIntPtr]::Zero)
|
|
676
|
-
Start-Sleep -Milliseconds 45
|
|
677
|
-
[BC2]::mouse_event(${up}, 0, 0, 0, [UIntPtr]::Zero)
|
|
678
|
-
`);
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
async function wheel(dir) {
|
|
682
|
-
const delta = dir < 0 ? -240 : 240;
|
|
683
|
-
await ps(WHEEL_TPL.replace(/%%DELTA%%/g, String(delta)));
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
/* legacy op yüzeyi: koordinat tabanlı eski aksiyonlar (cua-driver'sız mod) */
|
|
687
|
-
async function legacyAct(op, args) {
|
|
688
|
-
op = String(op || '').trim().toLowerCase();
|
|
689
|
-
const noDriver = (what) => ({
|
|
690
|
-
ok: false,
|
|
691
|
-
error: `${what} için cua-driver gerekli — kurulum: irm ${CUA_INSTALL_PS1} | iex` +
|
|
692
|
-
(INST.running ? ' (kurulum arka planda sürüyor — birkaç dakika sonra tekrar dene)' : '') +
|
|
693
|
-
(INST.failed ? ' (önceki otomatik kurulum başarısız oldu)' : ''),
|
|
694
|
-
});
|
|
695
|
-
switch (op) {
|
|
696
|
-
case 'capture':
|
|
697
|
-
case 'list_apps':
|
|
698
|
-
case 'focus_app':
|
|
699
|
-
return noDriver('capture/list_apps/focus_app');
|
|
700
|
-
case 'drag':
|
|
701
|
-
return noDriver('drag');
|
|
702
|
-
case 'click':
|
|
703
|
-
case 'dblclick':
|
|
704
|
-
case 'rightclick':
|
|
705
|
-
case 'type': {
|
|
706
|
-
if (args.element != null) return noDriver('element tıklama');
|
|
707
|
-
break;
|
|
708
|
-
}
|
|
709
|
-
default:
|
|
710
|
-
break;
|
|
711
|
-
}
|
|
712
|
-
if (op === 'screen_size') {
|
|
713
|
-
const s = await legacyScreenSize();
|
|
714
|
-
return { ok: true, ...s };
|
|
715
|
-
}
|
|
716
|
-
await pace();
|
|
717
|
-
const x = Math.max(0, Math.round(Number(args.x) || 0));
|
|
718
|
-
const y = Math.max(0, Math.round(Number(args.y) || 0));
|
|
719
|
-
switch (op) {
|
|
720
|
-
case 'move':
|
|
721
|
-
await mouseMove(x, y);
|
|
722
|
-
return { ok: true };
|
|
723
|
-
case 'click':
|
|
724
|
-
if (args.x == null && args.y == null) return { ok: false, error: 'x/y gerekli' };
|
|
725
|
-
await mouseMove(x, y);
|
|
726
|
-
await clickSimple('left');
|
|
727
|
-
return { ok: true, at: { x, y } };
|
|
728
|
-
case 'dblclick':
|
|
729
|
-
if (args.x == null && args.y == null) return { ok: false, error: 'x/y gerekli' };
|
|
730
|
-
await mouseMove(x, y);
|
|
731
|
-
await clickSimple('left');
|
|
732
|
-
await sleep(70);
|
|
733
|
-
await clickSimple('left');
|
|
734
|
-
return { ok: true, at: { x, y } };
|
|
735
|
-
case 'rightclick':
|
|
736
|
-
if (args.x == null && args.y == null) return { ok: false, error: 'x/y gerekli' };
|
|
737
|
-
await mouseMove(x, y);
|
|
738
|
-
await clickSimple('right');
|
|
739
|
-
return { ok: true, at: { x, y } };
|
|
740
|
-
case 'type': {
|
|
741
|
-
const text = String(args.text ?? '').slice(0, 2000);
|
|
742
|
-
if (!text) return { ok: false, error: 'boş metin' };
|
|
743
|
-
await ps(buildTyperScript(text));
|
|
744
|
-
return { ok: true, chars: text.length };
|
|
745
|
-
}
|
|
746
|
-
case 'key': {
|
|
747
|
-
const combo = String(args.keys || args.combo || '').trim().slice(0, 40);
|
|
748
|
-
if (!combo) return { ok: false, error: 'tuş gerekli' };
|
|
749
|
-
await ps(buildKeyScript(combo));
|
|
750
|
-
return { ok: true, key: combo };
|
|
751
|
-
}
|
|
752
|
-
case 'scroll': {
|
|
753
|
-
if (args.direction) {
|
|
754
|
-
const dy = args.direction === 'down' ? Number(args.amount) || 3 : args.direction === 'up' ? -(Number(args.amount) || 3) : 0;
|
|
755
|
-
if (!dy) return { ok: false, error: 'dikey scroll için up/down gerekli' };
|
|
756
|
-
await mouseMove(x || 640, y || 360);
|
|
757
|
-
const clicks = Math.abs(dy) * 3;
|
|
758
|
-
for (let i = 0; i < clicks; i++) await wheel(dy > 0 ? 1 : -1);
|
|
759
|
-
return { ok: true, direction: args.direction, amount: Math.abs(dy) };
|
|
760
|
-
}
|
|
761
|
-
const dy = clamp(Math.round(Number(args.dy) || 0), -10, 10);
|
|
762
|
-
if (!dy) return { ok: false, error: 'direction veya dy gerekli (-10..10)' };
|
|
763
|
-
await mouseMove(x || 640, y || 360);
|
|
764
|
-
const clicks = Math.abs(dy) * 3;
|
|
765
|
-
for (let i = 0; i < clicks; i++) {
|
|
766
|
-
await wheel(dy > 0 ? 1 : -1);
|
|
767
|
-
}
|
|
768
|
-
return { ok: true, dy };
|
|
769
|
-
}
|
|
770
|
-
default:
|
|
771
|
-
return { ok: false, error: `bilinmeyen op: ${op}` };
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/* ---------- genel giriş ---------- */
|
|
776
|
-
|
|
777
|
-
/**
|
|
778
|
-
* Hermes-SKILL.md aksiyon kümesi:
|
|
779
|
-
* capture mode=som|vision|ax app=… → numaralı element index'i (+görsel)
|
|
780
|
-
* click element=N | x,y button=left|right|middle
|
|
781
|
-
* dblclick / rightclick element=N | x,y
|
|
782
|
-
* type text="…" element=N | x,y
|
|
783
|
-
* key keys="ctrl+s" | "enter"
|
|
784
|
-
* scroll direction=up|down|left|right amount=1..50 element=N | x,y
|
|
785
|
-
* drag from_element/to_element | from_x/y → to_x/y
|
|
786
|
-
* wait seconds=0.5
|
|
787
|
-
* list_apps (argüman yok)
|
|
788
|
-
* focus_app app=… [raise_window=true]
|
|
789
|
-
* move x,y (yalnız legacy — gerçek cursor'u oynatır)
|
|
790
|
-
* State-changing aksiyonlarda capture_after=true → aynı hedefte doğrulama
|
|
791
|
-
* capture'ı tek çağrıda döndürür.
|
|
792
|
-
*/
|
|
793
|
-
async function act(op, args = {}) {
|
|
794
|
-
op = String(op || '').trim().toLowerCase();
|
|
795
|
-
args = args || {};
|
|
796
|
-
if (op === 'wait') {
|
|
797
|
-
const ms = clamp(
|
|
798
|
-
args.seconds != null ? Number(args.seconds) * 1000 : args.ms != null ? Number(args.ms) : 500,
|
|
799
|
-
0, 30000
|
|
800
|
-
);
|
|
801
|
-
await sleep(ms);
|
|
802
|
-
return { ok: true, op: 'wait', waited: Math.round(ms) };
|
|
803
|
-
}
|
|
804
|
-
const g = guard(op, args);
|
|
805
|
-
if (g) return g;
|
|
806
|
-
const stateChanging = ['click', 'dblclick', 'rightclick', 'type', 'key', 'scroll', 'drag'];
|
|
807
|
-
try {
|
|
808
|
-
if (await ensure()) {
|
|
809
|
-
const r = await driverAct(op, args);
|
|
810
|
-
if (r !== null) {
|
|
811
|
-
if (stateChanging.includes(op) && args.capture_after) await attachCapture(r);
|
|
812
|
-
return r;
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
} catch (e) {
|
|
816
|
-
teardown(String((e && e.message) || e));
|
|
817
|
-
S.downUntil = Date.now() + RETRY_COOLDOWN;
|
|
818
|
-
/* sürücü koptu → legacy devam */
|
|
819
|
-
}
|
|
820
|
-
return legacyAct(op, args);
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
/* Çözünürlük: sürücü varsa driver (logical points), yoksa PowerShell (piksel) */
|
|
824
|
-
async function screenSize() {
|
|
825
|
-
try {
|
|
826
|
-
if (await ensure()) {
|
|
827
|
-
const r = await call('get_screen_size', {}, 10000);
|
|
828
|
-
const s = r.structured || {};
|
|
829
|
-
const w = Number(s.width || s.w || 0);
|
|
830
|
-
const h = Number(s.height || s.h || 0);
|
|
831
|
-
if (w && h) return { w, h };
|
|
832
|
-
}
|
|
833
|
-
} catch {}
|
|
834
|
-
return legacyScreenSize();
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
module.exports = { act, screenSize, clickSimple, mouseMove, probe, autoInstall };
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/* Beast Computer Use (#4): Windows GUI otomasyonu — ekran görüntüsü +
|
|
4
|
+
fare/klavye kontrolü. Ek paket YOK: Electron nativeImage ekranı verir,
|
|
5
|
+
mouse/klavye PowerShell üzerinden user32 API ile sürülür.
|
|
6
|
+
|
|
7
|
+
Güvenlik: yalnızca engine araç çağrılarıyla çalışır; koordinat sınırlı,
|
|
8
|
+
hız limiting'li (aksiyon arası min gecikme) ve yazma metni kırpılır. */
|
|
9
|
+
|
|
10
|
+
const { execFile } = require('child_process');
|
|
11
|
+
|
|
12
|
+
/* aksiyonlar arası minimum boşluk — wx'i art arda tıklarla donatmamak için */
|
|
13
|
+
let lastActionAt = 0;
|
|
14
|
+
function pace() {
|
|
15
|
+
const wait = 120 - (Date.now() - lastActionAt);
|
|
16
|
+
lastActionAt = Date.now();
|
|
17
|
+
if (wait > 0) return new Promise((r) => setTimeout(r, wait));
|
|
18
|
+
return Promise.resolve();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function ps(script) {
|
|
22
|
+
return new Promise((resolve, reject) => {
|
|
23
|
+
execFile(
|
|
24
|
+
'powershell.exe',
|
|
25
|
+
['-NoProfile', '-NonInteractive', '-Command', script],
|
|
26
|
+
{ timeout: 20000, windowsHide: true },
|
|
27
|
+
(err, stdout) => (err ? reject(err) : resolve(String(stdout || '').trim()))
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Sanal masaüstü çözünürlüğü */
|
|
33
|
+
async function screenSize() {
|
|
34
|
+
const out = await ps(
|
|
35
|
+
'Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width.ToString() + "x" + [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Height'
|
|
36
|
+
);
|
|
37
|
+
const m = out.match(/(\d+)x(\d+)/);
|
|
38
|
+
if (!m) throw new Error('çözünürlük alınamadı');
|
|
39
|
+
return { w: Number(m[1]), h: Number(m[2]) };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Aksiyon uygula. op:
|
|
44
|
+
* click x y → sol tık
|
|
45
|
+
* dblclick x y → çift tık
|
|
46
|
+
* rightclick x y → sağ tık
|
|
47
|
+
* move x y → imleci taşı
|
|
48
|
+
* type text → klavyeyle yaz (Unicode destekli, SendInput WM_CHAR benzeri)
|
|
49
|
+
* key combo → "enter", "ctrl+s", "alt+tab", "win" ...
|
|
50
|
+
* scroll dx dy → tekerlek
|
|
51
|
+
*/
|
|
52
|
+
async function act(op, args = {}) {
|
|
53
|
+
await pace();
|
|
54
|
+
const x = Math.max(0, Math.round(Number(args.x) || 0));
|
|
55
|
+
const y = Math.max(0, Math.round(Number(args.y) || 0));
|
|
56
|
+
switch (op) {
|
|
57
|
+
case 'move':
|
|
58
|
+
await mouseMove(x, y);
|
|
59
|
+
return { ok: true };
|
|
60
|
+
case 'click':
|
|
61
|
+
await mouseMove(x, y);
|
|
62
|
+
await clickSimple('left');
|
|
63
|
+
return { ok: true, at: { x, y } };
|
|
64
|
+
case 'dblclick':
|
|
65
|
+
await mouseMove(x, y);
|
|
66
|
+
await clickSimple('left');
|
|
67
|
+
await sleep(70);
|
|
68
|
+
await clickSimple('left');
|
|
69
|
+
return { ok: true, at: { x, y } };
|
|
70
|
+
case 'rightclick':
|
|
71
|
+
await mouseMove(x, y);
|
|
72
|
+
await clickSimple('right');
|
|
73
|
+
return { ok: true, at: { x, y } };
|
|
74
|
+
case 'type': {
|
|
75
|
+
const text = String(args.text ?? '').slice(0, 2000);
|
|
76
|
+
if (!text) return { ok: false, error: 'boş metin' };
|
|
77
|
+
/* Add-Type SendInput tabanlı Unicode typer */
|
|
78
|
+
await ps(buildTyperScript(text));
|
|
79
|
+
return { ok: true, chars: text.length };
|
|
80
|
+
}
|
|
81
|
+
case 'key': {
|
|
82
|
+
const combo = String(args.combo || args.key || '').trim().slice(0, 40);
|
|
83
|
+
if (!combo) return { ok: false, error: 'tuş gerekli' };
|
|
84
|
+
await ps(buildKeyScript(combo));
|
|
85
|
+
return { ok: true, key: combo };
|
|
86
|
+
}
|
|
87
|
+
case 'scroll': {
|
|
88
|
+
const dy = Math.max(-10, Math.min(10, Math.round(Number(args.dy) || 0)));
|
|
89
|
+
if (!dy) return { ok: false, error: 'dy gerekli (-10..10)' };
|
|
90
|
+
await mouseMove(x || 640, y || 360);
|
|
91
|
+
const clicks = Math.abs(dy) * 3;
|
|
92
|
+
for (let i = 0; i < clicks; i++) {
|
|
93
|
+
await wheel(dy > 0 ? 1 : -1); // pozitif dy = aşağı
|
|
94
|
+
}
|
|
95
|
+
return { ok: true, dy };
|
|
96
|
+
}
|
|
97
|
+
default:
|
|
98
|
+
return { ok: false, error: `bilinmeyen op: ${op}` };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function sleep(ms) {
|
|
103
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/* ---------- PowerShell snippet üreticileri ---------- */
|
|
107
|
+
|
|
108
|
+
const MOUSE_MOVE_TPL = `
|
|
109
|
+
Add-Type @"
|
|
110
|
+
using System;
|
|
111
|
+
using System.Runtime.InteropServices;
|
|
112
|
+
public class BM {
|
|
113
|
+
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
|
114
|
+
}
|
|
115
|
+
"@
|
|
116
|
+
[BM]::SetCursorPos(%%X%%, %%Y%%) | Out-Null
|
|
117
|
+
`;
|
|
118
|
+
|
|
119
|
+
const WHEEL_TPL = `
|
|
120
|
+
Add-Type @"
|
|
121
|
+
using System;
|
|
122
|
+
using System.Runtime.InteropServices;
|
|
123
|
+
public class BW {
|
|
124
|
+
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
|
|
125
|
+
}
|
|
126
|
+
"@
|
|
127
|
+
[BW]::mouse_event(0x0800, 0, 0, %%DELTA%%, [UIntPtr]::Zero)
|
|
128
|
+
`;
|
|
129
|
+
|
|
130
|
+
/* Unicode güvenli yazım: SendInput yerine pano + Ctrl+V fallback'u en sağlamı.
|
|
131
|
+
Panoyu geri yüklemek yerine tek seferlik kullanıp bırakıyoruz (hız öncelik). */
|
|
132
|
+
function buildTyperScript(text) {
|
|
133
|
+
const b64 = Buffer.from(text, 'utf16le').toString('base64');
|
|
134
|
+
return `
|
|
135
|
+
$t = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${b64}"))
|
|
136
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
137
|
+
[System.Windows.Forms.Clipboard]::SetText($t)
|
|
138
|
+
Start-Sleep -Milliseconds 80
|
|
139
|
+
[System.Windows.Forms.SendKeys]::SendWait("^v")
|
|
140
|
+
`;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/* Tuş kombosu: enter/tab/esc/f1-12, ctrl/alt/shift+x, win vb. */
|
|
144
|
+
function buildKeyScript(combo) {
|
|
145
|
+
const map = {
|
|
146
|
+
enter: '{ENTER}', tab: '{TAB}', esc: '{ESC}', escape: '{ESC}',
|
|
147
|
+
backspace: '{BACKSPACE}', delete: '{DELETE}', del: '{DELETE}',
|
|
148
|
+
home: '{HOME}', end: '{END}', pgup: '{PGUP}', pgdn: '{PGDN}',
|
|
149
|
+
up: '{UP}', down: '{DOWN}', left: '{LEFT}', right: '{RIGHT}',
|
|
150
|
+
space: ' ', win: '^({ESC})',
|
|
151
|
+
};
|
|
152
|
+
let out = '';
|
|
153
|
+
const parts = String(combo).toLowerCase().split('+').map((p) => p.trim());
|
|
154
|
+
let mods = '';
|
|
155
|
+
for (const p of parts) {
|
|
156
|
+
if (p === 'ctrl') mods += '^';
|
|
157
|
+
else if (p === 'alt') mods += '%';
|
|
158
|
+
else if (p === 'shift') mods += '+';
|
|
159
|
+
else if (p === 'win') out += ''; // win ayrı ele
|
|
160
|
+
else if (/^f\d{1,2}$/.test(p)) out += '{' + p.toUpperCase() + '}';
|
|
161
|
+
else if (map[p]) out += map[p];
|
|
162
|
+
else out += p.slice(0, 1).toUpperCase(); // tek harf/tuş
|
|
163
|
+
}
|
|
164
|
+
const seq = mods && out ? mods + '(' + out + ')' : mods + out;
|
|
165
|
+
const b64 = Buffer.from(seq, 'utf16le').toString('base64');
|
|
166
|
+
return `
|
|
167
|
+
$s = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${b64}"))
|
|
168
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
169
|
+
[System.Windows.Forms.SendKeys]::SendWait($s)
|
|
170
|
+
`;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function mouseMove(x, y) {
|
|
174
|
+
await ps(MOUSE_MOVE_TPL.replace(/%%X%%/g, String(x)).replace(/%%Y%%/g, String(y)));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/* down+up tek script'te — karışık replace zinciri yerine bu kullanılır */
|
|
178
|
+
async function clickSimple(button) {
|
|
179
|
+
const down = button === 'right' ? 0x0008 : 0x0002;
|
|
180
|
+
const up = button === 'right' ? 0x0010 : 0x0004;
|
|
181
|
+
await ps(`
|
|
182
|
+
Add-Type @"
|
|
183
|
+
using System;
|
|
184
|
+
using System.Runtime.InteropServices;
|
|
185
|
+
public class BC2 {
|
|
186
|
+
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
|
|
187
|
+
}
|
|
188
|
+
"@
|
|
189
|
+
[BC2]::mouse_event(${down}, 0, 0, 0, [UIntPtr]::Zero)
|
|
190
|
+
Start-Sleep -Milliseconds 45
|
|
191
|
+
[BC2]::mouse_event(${up}, 0, 0, 0, [UIntPtr]::Zero)
|
|
192
|
+
`);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async function wheel(dir) {
|
|
196
|
+
const delta = dir < 0 ? -240 : 240; // WHEEL_DELTA=120 katları; negatif=yukarı görüntüde aşağı kayar bazı app'lerde
|
|
197
|
+
await ps(WHEEL_TPL.replace(/%%DELTA%%/g, String(delta)));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
module.exports = { act, screenSize, clickSimple, mouseMove };
|
package/src/agent/engine.js
CHANGED
|
@@ -2006,7 +2006,7 @@ class Engine {
|
|
|
2006
2006
|
'- mt5_ea action:"note": grafiğe kısa plan metni + yatay seviye çizgileri yazar (destek/direnç/SL/TP) — sonra tool__mt5_shot ile çekip seviyelerin doğru yerde olduğunu GÖRSEL doğrula.\n' +
|
|
2007
2007
|
'- computer_look: tüm masaüstü ekranı (MT5 dışı pencereler dahil); browser_screenshot: web grafikleri (TradingView/Investing) — browser_open ile aç.\n' +
|
|
2008
2008
|
'- Görsel göremiyorsan (metin-model) ocr_read (source:"screen"/"browser") ile grafikteki fiyat/seviyeleri metne çevir.\n' +
|
|
2009
|
-
'- Grafikte gezinme/ölçek: computer_act ile timeframe, scroll, zoom (
|
|
2009
|
+
'- Grafikte gezinme/ölçek: computer_act ile timeframe, scroll, zoom (yerel fare/klavye sürer — imleci oynatır; kısa tut).\n' +
|
|
2010
2010
|
'- EKİPLE GÖRSEL PAYLAŞIM: chart PNG\'sini agent_dm image:true (son görüntün) ya da image:"<dosya yolu>" (ör. tool__mt5_shot dönen path) ile trader/ekibe gönder — karşı ajan görüntüyü GERÇEKTEN görür, AJAN DM panelinde açılır. Ortak kararlarda grafiği paylaşmak zorunlu disiplindir.\n' +
|
|
2011
2011
|
'- PARALEL EKİP AKIŞI: teknik ajan seviyeleri hesaplar → mt5_ea note ile grafiğe çizer; görsel ajan tool__mt5_shot ile çekip TEYİT eder ve agent_dm ile görseli trader\'a atar; risk ajanı aynı görsel üzerinden SL/TP yerleşimini denetler; trader kararı işleme çevirir. Aynı sembolde koşan ajanlar tek DM grubunda (agent_dm group) toplanır — çelişen hareket etme.\n' +
|
|
2012
2012
|
'- EKSİK ARAÇ: özel grafik aracı gerekiyorsa (ör. çoklu timeframe tek görüntü, indikatör paneli okuma) VAR SAYMA — tool_request ile TOOL botuna yazdır (asenkron çalışır; rapor sohbete düşer, araç doğrulanınca tool__<ad> olarak ANINDA çağrılır); küçük aracı skill("tool-yazma") prosedürüyle kendin de yazabilirsin.\n' +
|
|
@@ -6412,36 +6412,19 @@ const TOOLS = [
|
|
|
6412
6412
|
function: {
|
|
6413
6413
|
name: 'computer_act',
|
|
6414
6414
|
description:
|
|
6415
|
-
'
|
|
6415
|
+
'Control mouse/keyboard on the user\u2019s Windows desktop (GUI automation). Ops: click{x,y}, dblclick{x,y}, rightclick{x,y}, move{x,y}, type{text}, key{combo e.g. "ctrl+s","enter","alt+tab"}, scroll{x,y,dy}. Look first with computer_look, act step by step, look again after acting.',
|
|
6416
6416
|
parameters: {
|
|
6417
6417
|
type: 'object',
|
|
6418
6418
|
properties: {
|
|
6419
6419
|
op: {
|
|
6420
6420
|
type: 'string',
|
|
6421
|
-
enum: ['
|
|
6422
|
-
description: 'capture first (som/vision/ax); click/type/etc act by element or window-local pixel; wait pauses; list_apps enumerates windows; focus_app selects input target (raise_window=true steals focus \u2014 only on explicit request); move = legacy real-cursor move',
|
|
6421
|
+
enum: ['click', 'dblclick', 'rightclick', 'move', 'type', 'key', 'scroll'],
|
|
6423
6422
|
},
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
text: { type: 'string', description: 'for op=type (max 2000 chars); NEVER type passwords or secrets' },
|
|
6430
|
-
keys: { type: 'string', description: 'for op=key: "enter", "ctrl+s", "alt+tab"' },
|
|
6431
|
-
direction: { type: 'string', enum: ['up', 'down', 'left', 'right'], description: 'for op=scroll' },
|
|
6432
|
-
amount: { type: 'number', description: 'for op=scroll: wheel notches 1..50 (default 3)' },
|
|
6433
|
-
from_element: { type: 'number', description: 'for op=drag' },
|
|
6434
|
-
to_element: { type: 'number', description: 'for op=drag' },
|
|
6435
|
-
from_x: { type: 'number', description: 'for op=drag (window-local pixels)' },
|
|
6436
|
-
from_y: { type: 'number', description: 'for op=drag (window-local pixels)' },
|
|
6437
|
-
to_x: { type: 'number', description: 'for op=drag (window-local pixels)' },
|
|
6438
|
-
to_y: { type: 'number', description: 'for op=drag (window-local pixels)' },
|
|
6439
|
-
seconds: { type: 'number', description: 'for op=wait (max 30)' },
|
|
6440
|
-
mode: { type: 'string', enum: ['som', 'vision', 'ax'], description: 'for op=capture: som = screenshot + numbered elements (default), vision = screenshot only, ax = element tree only' },
|
|
6441
|
-
capture_after: { type: 'boolean', description: 're-capture the target right after this action (verify in the same call)' },
|
|
6442
|
-
delivery_mode: { type: 'string', enum: ['background', 'foreground'], description: 'background (default) never raises the window; foreground is the last-resort rung \u2014 use ONLY after effect:"suspected_noop" or background_unavailable' },
|
|
6443
|
-
dy: { type: 'number', description: 'legacy scroll fallback: positive = down (-10..10)' },
|
|
6444
|
-
combo: { type: 'string', description: 'legacy alias for keys' },
|
|
6423
|
+
x: { type: 'number' },
|
|
6424
|
+
y: { type: 'number' },
|
|
6425
|
+
text: { type: 'string', description: 'for op=type (max 2000 chars)' },
|
|
6426
|
+
combo: { type: 'string', description: 'for op=key: "enter", "ctrl+s", "alt+tab", "win"' },
|
|
6427
|
+
dy: { type: 'number', description: 'for op=scroll: positive = down (-10..10)' },
|
|
6445
6428
|
},
|
|
6446
6429
|
required: ['op'],
|
|
6447
6430
|
},
|
package/src/agent/watext.js
CHANGED
|
@@ -70,7 +70,7 @@ const TOOL_LABELS = {
|
|
|
70
70
|
kb_search: (a) => `Bilgi ara: ${String(a.query || '').slice(0, 60)}`,
|
|
71
71
|
memory_hygiene: () => 'Hafıza temizliği',
|
|
72
72
|
computer_look: () => 'Ekranı görüntülüyor',
|
|
73
|
-
computer_act: (a) => `Bilgisayar kullanıyor: ${a.op || ''}${a.
|
|
73
|
+
computer_act: (a) => `Bilgisayar kullanıyor: ${a.op || ''}${a.combo ? ' ' + a.combo : ''}`,
|
|
74
74
|
};
|
|
75
75
|
|
|
76
76
|
function waToolLine(name, args) {
|
package/src/main.js
CHANGED
|
@@ -6785,33 +6785,12 @@ ipcMain.handle('install:status', async () => {
|
|
|
6785
6785
|
rows.push({ id: 'python', name: 'Python — betikler / web arama', state: pyVer ? 'ok' : 'optional', detail: pyVer ? 'v' + pyVer : 'sistemde bulunamadı — opsiyonel' });
|
|
6786
6786
|
}
|
|
6787
6787
|
|
|
6788
|
-
/* 7)
|
|
6789
|
-
{
|
|
6790
|
-
const st = await require('./agent/computeruse').probe().catch(() => ({ installed: false }));
|
|
6791
|
-
rows.push({
|
|
6792
|
-
id: 'cua-driver',
|
|
6793
|
-
name: 'cua-driver — Computer Use (masaüstü otomasyonu)',
|
|
6794
|
-
state: st.installed ? 'ok' : (st.installing ? 'loading' : (st.failed ? 'failed' : 'missing')),
|
|
6795
|
-
detail: st.installed
|
|
6796
|
-
? (st.version ? 'kurulu — ' + st.version : 'kurulu')
|
|
6797
|
-
: st.installing
|
|
6798
|
-
? 'cua.ai sürücüsü arka planda kuruluyor'
|
|
6799
|
-
: (st.failed
|
|
6800
|
-
? 'kurulum başarısız — elle: irm https://cua.ai/driver/install.ps1 | iex'
|
|
6801
|
-
: 'ilk Computer Use kullanımında arka planda kurulur'),
|
|
6802
|
-
...(st.installing ? pctFields('cua-driver') : {}),
|
|
6803
|
-
});
|
|
6804
|
-
}
|
|
6805
|
-
|
|
6806
|
-
/* 8) Edge TTS (bulut) */
|
|
6788
|
+
/* 7) Edge TTS (bulut) */
|
|
6807
6789
|
rows.push({ id: 'edge', name: 'Edge TTS — seslendirme', state: 'cloud', detail: 'bulut — kurulum gerekmez' });
|
|
6808
6790
|
|
|
6809
6791
|
return rows;
|
|
6810
6792
|
});
|
|
6811
6793
|
|
|
6812
|
-
/* cua-driver kurulumunu şimdi başlat (Kurulum sekmesi otomatiği + elle tetikleme) */
|
|
6813
|
-
ipcMain.handle('cua:install', () => require('./agent/computeruse').autoInstall());
|
|
6814
|
-
|
|
6815
6794
|
/* SQUEEZE (kendi token sıkıştırıcımız): aç/kapa + durum + istatistik */
|
|
6816
6795
|
ipcMain.handle('squeeze:toggle', (_e, on) => {
|
|
6817
6796
|
settings.squeeze = { ...(settings.squeeze || {}), enabled: !!on };
|
package/src/preload.js
CHANGED
|
@@ -65,7 +65,6 @@ contextBridge.exposeInMainWorld('beast', {
|
|
|
65
65
|
sttStatus: () => ipcRenderer.invoke('stt:status'),
|
|
66
66
|
sttPrefetchNow: () => ipcRenderer.invoke('stt:prefetch'),
|
|
67
67
|
installStatus: () => ipcRenderer.invoke('install:status'),
|
|
68
|
-
cuaInstall: () => ipcRenderer.invoke('cua:install'),
|
|
69
68
|
embedPrefetch: () => ipcRenderer.invoke('embed:prefetch'),
|
|
70
69
|
sttLangSet: (lang) => ipcRenderer.invoke('stt:lang:set', lang),
|
|
71
70
|
updateStatus: () => ipcRenderer.invoke('update:status'),
|
package/src/renderer/renderer.js
CHANGED
|
@@ -2472,7 +2472,6 @@ async function renderInstallPane() {
|
|
|
2472
2472
|
loading: ['İNİYOR…', '#d9a441'],
|
|
2473
2473
|
partial: ['İNİYOR… (devam)', '#d9a441'],
|
|
2474
2474
|
missing: ['İNDİRİLİYOR…', '#d9a441'],
|
|
2475
|
-
failed: ['HATA', '#c0554d'],
|
|
2476
2475
|
cloud: ['BULUT', 'var(--muted)'],
|
|
2477
2476
|
optional: ['OPSİYONEL', 'var(--muted)'],
|
|
2478
2477
|
};
|
|
@@ -2492,10 +2491,6 @@ async function renderInstallPane() {
|
|
|
2492
2491
|
autoStarted.add('emb');
|
|
2493
2492
|
beast.embedPrefetch().catch(() => {});
|
|
2494
2493
|
}
|
|
2495
|
-
if (r.id === 'cua-driver' && r.state === 'missing') {
|
|
2496
|
-
autoStarted.add('cua-driver');
|
|
2497
|
-
beast.cuaInstall().catch(() => {});
|
|
2498
|
-
}
|
|
2499
2494
|
}
|
|
2500
2495
|
};
|
|
2501
2496
|
|