deepseek-computer-use 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/native/Driver.cs CHANGED
@@ -1,1906 +1,1906 @@
1
-
2
- using System;
3
- using System.Collections.Generic;
4
- using System.Diagnostics;
5
- using System.Drawing;
6
- using System.Drawing.Imaging;
7
- using System.IO;
8
- using System.Runtime.InteropServices;
9
- using System.Text;
10
- using System.Windows.Automation;
11
-
12
- // ---------------------------------------------------------------------------
13
- // Platform driver contract. Windows is the only implementation today; the
14
- // interface exists so macOS (AX + CGEvent) and Linux (AT-SPI) drivers can be
15
- // added without touching the DSH plugin layer or the stdio protocol.
16
- // ---------------------------------------------------------------------------
17
- public interface IPlatformDriver : IDisposable
18
- {
19
- string PlatformName { get; }
20
- Dictionary<string, object> Ping(Dictionary<string, object> p);
21
- Dictionary<string, object> Hello(Dictionary<string, object> p);
22
- Dictionary<string, object> RestoreCursor(Dictionary<string, object> p);
23
- void Maintenance();
24
- Dictionary<string, object> Close(Dictionary<string, object> p);
25
- Dictionary<string, object> ListWindows(Dictionary<string, object> p);
26
- Dictionary<string, object> ListApps(Dictionary<string, object> p);
27
- Dictionary<string, object> GetWindow(Dictionary<string, object> p);
28
- Dictionary<string, object> LaunchApp(Dictionary<string, object> p);
29
- Dictionary<string, object> GetWindowState(Dictionary<string, object> p);
30
- Dictionary<string, object> Click(Dictionary<string, object> p);
31
- Dictionary<string, object> PressKey(Dictionary<string, object> p);
32
- Dictionary<string, object> TypeText(Dictionary<string, object> p);
33
- Dictionary<string, object> Scroll(Dictionary<string, object> p);
34
- Dictionary<string, object> Drag(Dictionary<string, object> p);
35
- Dictionary<string, object> SetValue(Dictionary<string, object> p);
36
- Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p);
37
- Dictionary<string, object> ActivateWindow(Dictionary<string, object> p);
38
- Dictionary<string, object> OverlayShow(Dictionary<string, object> p);
39
- Dictionary<string, object> OverlayHide(Dictionary<string, object> p);
40
- Dictionary<string, object> Configure(Dictionary<string, object> p);
41
- Dictionary<string, object> ApproveApp(Dictionary<string, object> p);
42
- Dictionary<string, object> EndTurn(Dictionary<string, object> p);
43
- Dictionary<string, object> InvalidateState(Dictionary<string, object> p);
44
- Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p);
45
- Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p);
46
- }
47
-
48
- /// Per-request time budget. Codex sends x-oai-cua-request-budget-ms with every call
49
- /// and the helper honours it; here the budget bounds capture waits and is reported
50
- /// back when a request overruns so the client and the helper agree on the deadline.
51
- public static class Budget
52
- {
53
- static long _deadlineTicks;
54
- static string _cancelFile;
55
-
56
- public static void Begin(long ms, string cancelFile)
57
- {
58
- _deadlineTicks = ms > 0 ? DateTime.UtcNow.Ticks + ms * TimeSpan.TicksPerMillisecond : 0;
59
- _cancelFile = cancelFile;
60
- }
61
-
62
- public static void Check()
63
- {
64
- if (_cancelFile != null && File.Exists(_cancelFile))
65
- {
66
- string reason = "cancelled";
67
- try { reason = File.ReadAllText(_cancelFile).Trim(); } catch { }
68
- if (reason != "turn_ended" && reason != "timeout") reason = "cancelled";
69
- throw new ToolError(reason, "the native request was stopped; take a fresh observation before continuing");
70
- }
71
- if (Expired()) throw new ToolError("timeout", "the native request exceeded its time budget");
72
- }
73
-
74
- public static long RemainingMs()
75
- {
76
- if (_deadlineTicks == 0) return long.MaxValue / 4;
77
- return (_deadlineTicks - DateTime.UtcNow.Ticks) / TimeSpan.TicksPerMillisecond;
78
- }
79
-
80
- public static bool Expired()
81
- {
82
- return _deadlineTicks != 0 && RemainingMs() <= 0;
83
- }
84
- }
85
-
86
- public class ToolError : Exception
87
- {
88
- public readonly string Code;
89
- public ToolError(string code, string message) : base(message) { Code = code; }
90
- }
91
-
92
- // ---------------------------------------------------------------------------
93
- // Windows driver: Win32 + UI Automation + PrintWindow.
94
- // ---------------------------------------------------------------------------
95
- public class WindowsDriver : IPlatformDriver
96
- {
97
- public string PlatformName { get { return "windows"; } }
98
-
99
- // ---- win32 ----
100
- delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
101
-
102
- [DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc cb, IntPtr lParam);
103
- [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
104
- [DllImport("user32.dll")] static extern bool IsWindow(IntPtr hWnd);
105
- [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowTextW(IntPtr hWnd, StringBuilder sb, int max);
106
- [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetClassNameW(IntPtr hWnd, StringBuilder sb, int max);
107
- [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
108
- [DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT r);
109
- [DllImport("user32.dll")] static extern bool GetClientRect(IntPtr hWnd, out RECT r);
110
- [DllImport("user32.dll")] static extern bool PrintWindow(IntPtr hWnd, IntPtr hdc, uint flags);
111
- [DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hWnd);
112
- [DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hWnd, int cmd);
113
- [DllImport("user32.dll")] static extern bool IsIconic(IntPtr hWnd);
114
- [DllImport("user32.dll")] static extern bool SetCursorPos(int x, int y);
115
- [DllImport("user32.dll")] static extern bool GetCursorPos(out POINT p);
116
- [DllImport("user32.dll")] static extern uint SendInput(uint n, INPUT[] inputs, int size);
117
- [DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
118
- [DllImport("user32.dll")] static extern IntPtr SetFocus(IntPtr hWnd);
119
- [DllImport("user32.dll")] static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach);
120
- [DllImport("kernel32.dll")] static extern uint GetCurrentThreadId();
121
- [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid);
122
- [DllImport("user32.dll")] static extern bool IsWindowEnabled(IntPtr hWnd);
123
- [DllImport("user32.dll")] static extern bool SystemParametersInfo(uint action, uint param, IntPtr pv, uint winIni);
124
- [DllImport("user32.dll")] static extern bool SwitchToThisWindow(IntPtr hWnd, bool fAltTab);
125
- [DllImport("user32.dll")] static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
126
- [DllImport("user32.dll")] static extern bool BringWindowToTop(IntPtr hWnd);
127
- [DllImport("user32.dll")] static extern IntPtr SetThreadDpiAwarenessContext(IntPtr ctx);
128
- [DllImport("user32.dll")] static extern int GetSystemMetrics(int index);
129
- [DllImport("user32.dll")] static extern bool SetProcessDPIAware();
130
- [DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT r, int size);
131
- [DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr hWnd, uint cmd);
132
- [DllImport("user32.dll")] static extern int GetWindowLong(IntPtr hWnd, int index);
133
-
134
- [StructLayout(LayoutKind.Sequential)] struct RECT { public int Left, Top, Right, Bottom; }
135
- [StructLayout(LayoutKind.Sequential)] struct POINT { public int X, Y; }
136
-
137
- [StructLayout(LayoutKind.Sequential)]
138
- struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
139
-
140
- [StructLayout(LayoutKind.Sequential)]
141
- struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
142
-
143
- [StructLayout(LayoutKind.Explicit)]
144
- struct INPUTUNION
145
- {
146
- [FieldOffset(0)] public MOUSEINPUT mi;
147
- [FieldOffset(0)] public KEYBDINPUT ki;
148
- }
149
-
150
- [StructLayout(LayoutKind.Sequential)]
151
- struct INPUT { public uint type; public INPUTUNION u; }
152
-
153
- const int INPUT_MOUSE = 0;
154
- const int INPUT_KEYBOARD = 1;
155
- const uint KEYEVENTF_KEYUP = 0x0002;
156
- const uint KEYEVENTF_UNICODE = 0x0004;
157
- const uint KEYEVENTF_EXTENDEDKEY = 0x0001;
158
- const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
159
- const uint MOUSEEVENTF_LEFTUP = 0x0004;
160
- const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
161
- const uint MOUSEEVENTF_RIGHTUP = 0x0010;
162
- const uint MOUSEEVENTF_MIDDLEDOWN = 0x0020;
163
- const uint MOUSEEVENTF_MIDDLEUP = 0x0040;
164
- const uint MOUSEEVENTF_WHEEL = 0x0800;
165
- const uint MOUSEEVENTF_HWHEEL = 0x1000;
166
- const int DWMWA_EXTENDED_FRAME_BOUNDS = 9;
167
-
168
- // ---- state ----
169
- readonly object _sync = new object();
170
- sealed class Snapshot
171
- {
172
- public long WindowId;
173
- public DateTime At;
174
- public Dictionary<long, AutomationElement> Elements = new Dictionary<long, AutomationElement>();
175
- public Dictionary<long, string> FallbackLabels = new Dictionary<long, string>();
176
- public List<object> Summary = new List<object>();
177
- }
178
-
179
- readonly Dictionary<long, Snapshot> _snapshots = new Dictionary<long, Snapshot>();
180
- int _snapshotTtlMs = 120000;
181
-
182
- /// A screenshot region the model was handed, so a later coordinate action can
183
- /// be interpreted in that exact screenshot space (Codex: screenshotId).
184
- sealed class ShotRegion
185
- {
186
- public long WindowId;
187
- public int OriginX;
188
- public int OriginY;
189
- public int Width;
190
- public int Height;
191
- /// Physical pixels per screenshot pixel. 1.0 for DPI-aware targets; >1 when
192
- /// the target is DPI-virtualized and PrintWindow returns an unscaled render.
193
- public double ScaleX = 1.0;
194
- public double ScaleY = 1.0;
195
- public DateTime At;
196
- }
197
-
198
- // In-use indicator ("DSH 正在使用您的电脑" + whale-girl). Codex draws an animated
199
- // border and a virtual cursor; this is the DSH equivalent, and Esc is the kill
200
- // switch: the overlay drops a cancel file that every input path checks first.
201
- Process _overlay;
202
- System.Threading.Timer _cursorWatchdog;
203
- DateTime _overlayUntil = DateTime.MinValue;
204
- string _overlayCancelFile;
205
- string _overlayText = "DSH 正在使用您的电脑";
206
- string _overlayHint = "Esc 取消";
207
- string _overlayAccent = "#3f7fd8";
208
- string _cursorFile; // the overlay polls this to draw the virtual cursor
209
- /// Turn token that the user interrupted with Esc. Codex persists the same
210
- /// fact as a marker file per conversation+turn; here the token travels with
211
- /// every input call, so the block ends exactly when the turn does.
212
- string _interruptedTurn = null;
213
- /// Apps the plugin approved through the DSH approval seam. Codex passes the same
214
- /// fact per request as x-oai-cua-approved-app; here the helper is the enforcer.
215
- readonly HashSet<string> _approvedApps = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
216
- bool _requireAppApproval = false;
217
- Process _audioProcess;
218
- string _audioPath;
219
- string _audioStopFile;
220
-
221
- readonly Dictionary<string, ShotRegion> _shots = new Dictionary<string, ShotRegion>();
222
- int _shotTtlMs = 120000;
223
- int _shotSeq = 0;
224
- readonly string _shotGeneration = Guid.NewGuid().ToString("N");
225
-
226
- const uint GW_OWNER = 4;
227
- const int GWL_STYLE = -16;
228
- const int WS_POPUP = unchecked((int)0x80000000);
229
-
230
- public WindowsDriver()
231
- {
232
- Maintenance(); // startup repair: a previous session may have died with cursors hidden
233
- try { SetThreadDpiAwarenessContext(new IntPtr(-4)); } catch { }
234
- try { SetProcessDPIAware(); } catch { }
235
- }
236
-
237
- public void Dispose()
238
- {
239
- if (_cursorWatchdog != null) { _cursorWatchdog.Dispose(); _cursorWatchdog = null; }
240
- StopOverlay();
241
- }
242
-
243
- // ---- approval plumbing ----
244
- public Dictionary<string, object> Configure(Dictionary<string, object> p)
245
- {
246
- _requireAppApproval = Json.GetBool(p, "requireAppApproval", _requireAppApproval);
247
- Dictionary<string, object> r = new Dictionary<string, object>();
248
- r["requireAppApproval"] = _requireAppApproval;
249
- return r;
250
- }
251
-
252
- public Dictionary<string, object> ApproveApp(Dictionary<string, object> p)
253
- {
254
- string app = Json.GetString(p, "app");
255
- if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "approve_app requires 'app'");
256
- lock (_sync) { _approvedApps.Add(app); }
257
- Dictionary<string, object> r = new Dictionary<string, object>();
258
- r["approved"] = app;
259
- r["count"] = (long)_approvedApps.Count;
260
- return r;
261
- }
262
-
263
- /// Turn boundary: drop per-turn state (element snapshots, screenshot regions, the
264
- /// Esc block, the indicator) but keep session-scoped app approvals.
265
- public Dictionary<string, object> EndTurn(Dictionary<string, object> p)
266
- {
267
- string ending = Json.GetString(p, "turnToken");
268
- if (ending != null) ClearInterrupt(ending);
269
- lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
270
- _interruptedTurn = null;
271
- StopOverlay();
272
- Dictionary<string, object> r = new Dictionary<string, object>();
273
- r["ended"] = true;
274
- r["approvedApps"] = (long)_approvedApps.Count;
275
- return r;
276
- }
277
-
278
- public Dictionary<string, object> InvalidateState(Dictionary<string, object> p)
279
- {
280
- lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
281
- StopOverlay();
282
- return new Dictionary<string, object> { { "invalidated", true } };
283
- }
284
-
285
- void AssertAppApproved(IntPtr h)
286
- {
287
- if (!_requireAppApproval) return;
288
- uint pid;
289
- GetWindowThreadProcessId(h, out pid);
290
- string id = ProcessPathOf(pid);
291
- lock (_sync) { if (_approvedApps.Contains(id)) return; }
292
- throw new ToolError("app_not_approved", "the Computer Use policy has not approved '" + id + "' yet. Approve the app through the DSH approval prompt before acting on its windows.");
293
- }
294
-
295
- // ---- audio recording (WASAPI loopback) ----
296
- public Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p)
297
- {
298
- bool envEnabled = Environment.GetEnvironmentVariable("DSH_CU_AUDIO") == "1";
299
- bool force = Json.GetBool(p, "force", false);
300
- // Codex treats computer audio as its own app ("computer-audio") and asks for a
301
- // separate approval before recording; the plugin passes that here.
302
- bool audioApproved = Json.GetString(p, "approvedApp") == "computer-audio";
303
- if (!envEnabled && !force && !audioApproved)
304
- throw new ToolError("audio_not_enabled", "computer audio recording is not enabled. Approve it (app \"computer-audio\") or set DSH_CU_AUDIO=1.");
305
- if (_audioProcess != null && !_audioProcess.HasExited)
306
- throw new ToolError("audio_running", "an audio recording is already running; stop it first");
307
-
308
- string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
309
- string exe = Path.Combine(dir, "audio", "dsh-cu-audio.exe");
310
- if (!File.Exists(exe)) throw new ToolError("audio_missing", "audio recorder not found at " + exe);
311
-
312
- long maxMs = Json.GetLong(p, "maxDurationMs", 60000);
313
- if (maxMs < 500) maxMs = 500;
314
- if (maxMs > 300000) maxMs = 300000; // Codex caps a single recording at 5 minutes
315
- _audioPath = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-" + Guid.NewGuid().ToString("N") + ".wav");
316
- _audioStopFile = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-stop-" + Guid.NewGuid().ToString("N") + ".flag");
317
- try { if (File.Exists(_audioStopFile)) File.Delete(_audioStopFile); } catch { }
318
-
319
- ProcessStartInfo psi = new ProcessStartInfo(exe, "--out \"" + _audioPath + "\" --max-ms " + maxMs + " --stop-file \"" + _audioStopFile + "\"");
320
- psi.UseShellExecute = false;
321
- psi.CreateNoWindow = true;
322
- psi.RedirectStandardOutput = true;
323
- psi.RedirectStandardError = true;
324
- _audioProcess = Process.Start(psi);
325
- System.Threading.Thread.Sleep(400);
326
- if (_audioProcess.HasExited)
327
- throw new ToolError("audio_not_started", "computer audio recording did not start in time (the recorder exited immediately)");
328
-
329
- Dictionary<string, object> r = new Dictionary<string, object>();
330
- r["started"] = true;
331
- r["filepath"] = _audioPath;
332
- r["maxDurationMs"] = maxMs;
333
- return r;
334
- }
335
-
336
- public Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p)
337
- {
338
- if (_audioProcess == null) throw new ToolError("audio_not_started", "no audio recording is running");
339
- try { if (_audioStopFile != null) File.WriteAllText(_audioStopFile, "stop"); } catch { }
340
- try { _audioProcess.WaitForExit(8000); } catch { }
341
- try { if (_audioProcess != null && !_audioProcess.HasExited) _audioProcess.Kill(); } catch { }
342
-
343
- string path = _audioPath;
344
- _audioProcess = null;
345
- _audioStopFile = null;
346
- _audioPath = null;
347
-
348
- byte[] audio = new byte[0];
349
- try { if (path != null && File.Exists(path)) audio = File.ReadAllBytes(path); } catch { }
350
-
351
- Dictionary<string, object> r = new Dictionary<string, object>();
352
- r["filepath"] = path;
353
- // Codex returns the raw WAV bytes plus a data URL; base64 is the JSON-safe form.
354
- r["bytes"] = Convert.ToBase64String(audio);
355
- r["byteLength"] = (long)audio.Length;
356
- r["data_url"] = audio.Length > 44 ? "data:audio/wav;base64," + Convert.ToBase64String(audio) : "";
357
- r["ok"] = audio.Length > 44;
358
- if (audio.Length <= 44) throw new ToolError("audio_not_recorded", "computer audio recording produced no samples");
359
- return r;
360
- }
361
-
362
- // ---- in-use indicator ----
363
- static string OverlayExePath()
364
- {
365
- string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location);
366
- return Path.Combine(dir ?? ".", "dsh-cu-overlay.exe");
367
- }
368
-
369
- public Dictionary<string, object> OverlayShow(Dictionary<string, object> p)
370
- {
371
- string text = Json.GetString(p, "text");
372
- if (!string.IsNullOrEmpty(text)) _overlayText = text;
373
- string hint = Json.GetString(p, "hint");
374
- if (!string.IsNullOrEmpty(hint)) _overlayHint = hint;
375
- string accent = Json.GetString(p, "accent");
376
- if (!string.IsNullOrEmpty(accent)) _overlayAccent = accent;
377
- EnsureOverlay(Json.GetLong(p, "ttlMs", 20000));
378
- Dictionary<string, object> r = new Dictionary<string, object>();
379
- r["visible"] = true;
380
- r["cancelFile"] = _overlayCancelFile;
381
- return r;
382
- }
383
-
384
- public Dictionary<string, object> OverlayHide(Dictionary<string, object> p)
385
- {
386
- StopOverlay();
387
- Dictionary<string, object> r = new Dictionary<string, object>();
388
- r["visible"] = false;
389
- return r;
390
- }
391
-
392
- void EnsureOverlay(long ttlMs)
393
- {
394
- string exe = OverlayExePath();
395
- if (!File.Exists(exe)) return;
396
- string mode = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR");
397
- if (mode != null && mode.Equals("off", StringComparison.OrdinalIgnoreCase)) { StopOverlay(); return; }
398
- string ttlEnv = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR_TTL_MS");
399
- long parsed;
400
- if (ttlEnv != null && long.TryParse(ttlEnv, out parsed) && parsed >= 1000) ttlMs = parsed;
401
- if (_overlay != null && !_overlay.HasExited && DateTime.UtcNow < _overlayUntil)
402
- {
403
- _overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
404
- return;
405
- }
406
- StopOverlay();
407
- _overlayCancelFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cancel-" + Guid.NewGuid().ToString("N") + ".flag");
408
- if (_cursorFile == null) _cursorFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cursor-" + Guid.NewGuid().ToString("N").Substring(0, 8) + ".flag");
409
- try { if (File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
410
- StringBuilder args = new StringBuilder();
411
- args.Append("--ttl-ms ").Append(ttlMs);
412
- args.Append(" --cancel-file \"").Append(_overlayCancelFile).Append('"');
413
- args.Append(" --text \"").Append(_overlayText).Append('"');
414
- args.Append(" --hint \"").Append(_overlayHint).Append('"');
415
- args.Append(" --owner-pid ").Append(Process.GetCurrentProcess().Id);
416
- args.Append(" --accent \"").Append(_overlayAccent).Append('"');
417
- if (_cursorFile != null) args.Append(" --cursor-file \"").Append(_cursorFile).Append('"');
418
- // Suppression is the default (the virtual cursor stands in for the real one);
419
- // DSH_CU_KEEP_CURSOR=1 keeps the real pointer visible instead.
420
- if (Environment.GetEnvironmentVariable("DSH_CU_KEEP_CURSOR") == "1") args.Append(" --no-suppress-cursor");
421
- try
422
- {
423
- ProcessStartInfo psi = new ProcessStartInfo(exe, args.ToString());
424
- psi.UseShellExecute = false;
425
- _overlay = Process.Start(psi);
426
- _overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
427
- // Codex runs a dedicated system-cursor-manager process that restores the
428
- // pointer the moment its parent dies. This timer gives the same guarantee
429
- // without a second exe: if the overlay vanishes with cursors hidden, the
430
- // pointer comes back within half a second, with no request needed.
431
- if (_cursorWatchdog == null)
432
- _cursorWatchdog = new System.Threading.Timer(delegate { try { Maintenance(); } catch { } }, null, 500, 500);
433
- }
434
- catch { _overlay = null; }
435
- }
436
-
437
- void StopOverlay()
438
- {
439
- try { if (_overlay != null && !_overlay.HasExited) _overlay.Kill(); } catch { }
440
- // Kill() skips the overlay's exit handlers, which is where it would restore the
441
- // system cursors. Restore them from here so a terminated overlay can never leave
442
- // the user without a pointer.
443
- RestoreCursors();
444
- try { File.Delete(OverlayCursorFlagPath()); } catch { }
445
- try { if (_overlayCancelFile != null && File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
446
- _overlay = null;
447
- _overlayUntil = DateTime.MinValue;
448
- }
449
-
450
- /// Esc on the overlay drops a cancel file; honour it before any other input.
451
- const string EscMessage = "Computer Use was stopped by the user with the physical Escape key. Stop your work, do not call further Computer Use tools in this turn, and send a final message noting that the user stopped Computer Use.";
452
-
453
- void CheckCancel(Dictionary<string, object> p)
454
- {
455
- Budget.Check();
456
- string token = p == null ? null : Json.GetString(p, "turnToken");
457
- if (token != null && (token == _interruptedTurn || InterruptMarkerExists(token)))
458
- throw new ToolError("turn_stopped", EscMessage);
459
- bool overlayExited = _overlay != null && _overlay.HasExited && _overlay.ExitCode == 130;
460
- bool cancelFile = _overlayCancelFile != null && File.Exists(_overlayCancelFile);
461
- if (!overlayExited && !cancelFile) return;
462
- if (cancelFile) { try { File.Delete(_overlayCancelFile); } catch { } }
463
- _interruptedTurn = token;
464
- PersistInterrupt(token);
465
- StopOverlay();
466
- Program.RequestedExitCode = 130; // Codex exits 130 when the user hits Esc
467
- throw new ToolError("cancelled_by_user", EscMessage);
468
- }
469
-
470
- const int ProtocolVersion = 2;
471
-
472
- static string InterruptMarkerPath(string token)
473
- {
474
- string root = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "interrupts");
475
- StringBuilder safe = new StringBuilder();
476
- if (token != null)
477
- {
478
- for (int i = 0; i < token.Length && safe.Length < 64; i++)
479
- {
480
- char ch = token[i];
481
- if (char.IsLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.') safe.Append(ch);
482
- else if (ch == ':') safe.Append('-');
483
- }
484
- }
485
- if (safe.Length == 0) safe.Append("turn");
486
- return Path.Combine(root, safe.ToString() + ".flag");
487
- }
488
-
489
- static void PersistInterrupt(string token)
490
- {
491
- try
492
- {
493
- string path = InterruptMarkerPath(token);
494
- Directory.CreateDirectory(Path.GetDirectoryName(path));
495
- File.WriteAllText(path, DateTime.UtcNow.ToString("o"));
496
- }
497
- catch { }
498
- }
499
-
500
- static bool InterruptMarkerExists(string token)
501
- {
502
- try { return File.Exists(InterruptMarkerPath(token)); } catch { return false; }
503
- }
504
-
505
- static void ClearInterrupt(string token)
506
- {
507
- try { File.Delete(InterruptMarkerPath(token)); } catch { }
508
- }
509
-
510
- /// Publish the pointer target for the virtual cursor. The overlay reads this
511
- /// file; a missing file simply means "no cursor to draw".
512
- void MarkCursor(int x, int y)
513
- {
514
- if (_cursorFile == null) return;
515
- try { File.WriteAllText(_cursorFile, x.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + y.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture)); } catch { }
516
- }
517
-
518
- /// Codex reports screenLocked instead of failing with an opaque automation error.
519
- static void AssertUnlocked()
520
- {
521
- IntPtr fg = GetForegroundWindow();
522
- if (fg == IntPtr.Zero) return;
523
- uint pid;
524
- GetWindowThreadProcessId(fg, out pid);
525
- string proc = "";
526
- try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
527
- if (proc.Equals("LockApp", StringComparison.OrdinalIgnoreCase) || proc.Equals("LogonUI", StringComparison.OrdinalIgnoreCase))
528
- throw new ToolError("screen_locked", "the Windows desktop is locked (LockApp is in the foreground), so Computer Use cannot interact with it. Ask the user to unlock the desktop and stop trying until they do.");
529
- }
530
-
531
- // ---- helpers ----
532
- static string TitleOf(IntPtr h)
533
- {
534
- StringBuilder sb = new StringBuilder(1024);
535
- GetWindowTextW(h, sb, sb.Capacity);
536
- return sb.ToString();
537
- }
538
-
539
- static string ClassOf(IntPtr h)
540
- {
541
- StringBuilder sb = new StringBuilder(256);
542
- GetClassNameW(h, sb, sb.Capacity);
543
- return sb.ToString();
544
- }
545
-
546
- static RECT BoundsOf(IntPtr h)
547
- {
548
- RECT r;
549
- try
550
- {
551
- if (DwmGetWindowAttribute(h, DWMWA_EXTENDED_FRAME_BOUNDS, out r, Marshal.SizeOf(typeof(RECT))) == 0 && r.Right > r.Left)
552
- return r;
553
- }
554
- catch { }
555
- GetWindowRect(h, out r);
556
- return r;
557
- }
558
-
559
- static Dictionary<string, object> RectDict(int x, int y, int w, int h)
560
- {
561
- Dictionary<string, object> d = new Dictionary<string, object>();
562
- d["x"] = x; d["y"] = y; d["width"] = w; d["height"] = h;
563
- return d;
564
- }
565
-
566
- static string ProcessPathOf(uint pid)
567
- {
568
- try
569
- {
570
- Process p = Process.GetProcessById((int)pid);
571
- try { return p.MainModule.FileName; }
572
- catch { return p.ProcessName; }
573
- }
574
- catch { return "unknown"; }
575
- }
576
-
577
- static string ProcessNameOf(uint pid)
578
- {
579
- try { return Process.GetProcessById((int)pid).ProcessName; }
580
- catch { return "unknown"; }
581
- }
582
-
583
- Dictionary<string, object> AppOf(uint pid)
584
- {
585
- Dictionary<string, object> app = new Dictionary<string, object>();
586
- string full = ProcessPathOf(pid);
587
- app["id"] = full;
588
- app["name"] = Path.GetFileNameWithoutExtension(full);
589
- app["displayName"] = ProcessNameOf(pid);
590
- app["pid"] = (long)pid;
591
- try
592
- {
593
- Process proc = Process.GetProcessById((int)pid);
594
- app["lastUsedDate"] = proc.StartTime.ToUniversalTime().ToString("o");
595
- }
596
- catch { }
597
- return app;
598
- }
599
-
600
- Dictionary<string, object> WindowDict(IntPtr h, uint pid, int z)
601
- {
602
- RECT r = BoundsOf(h);
603
- Dictionary<string, object> w = new Dictionary<string, object>();
604
- w["id"] = (long)h.ToInt64();
605
- w["title"] = TitleOf(h);
606
- w["app"] = AppOf(pid);
607
- w["bounds"] = RectDict(r.Left, r.Top, r.Right - r.Left, r.Bottom - r.Top);
608
- w["zIndex"] = z;
609
- w["className"] = ClassOf(h);
610
- return w;
611
- }
612
-
613
- List<IntPtr> EnumerateWindows(bool visibleOnly)
614
- {
615
- List<IntPtr> list = new List<IntPtr>();
616
- EnumWindows(delegate(IntPtr h, IntPtr lp)
617
- {
618
- if (!IsWindow(h)) return true;
619
- if (visibleOnly && !IsWindowVisible(h)) return true;
620
- uint pid;
621
- GetWindowThreadProcessId(h, out pid);
622
- if (pid == 0) return true;
623
- RECT r;
624
- GetWindowRect(h, out r);
625
- if (r.Right - r.Left < 80 || r.Bottom - r.Top < 60) return true;
626
- string title = TitleOf(h);
627
- if (title.Length == 0) return true;
628
- list.Add(h);
629
- return true;
630
- }, IntPtr.Zero);
631
- return list;
632
- }
633
-
634
- // ---- methods ----
635
- public Dictionary<string, object> Ping(Dictionary<string, object> p)
636
- {
637
- Dictionary<string, object> d = new Dictionary<string, object>();
638
- d["platform"] = PlatformName;
639
- d["pid"] = (long)Process.GetCurrentProcess().Id;
640
- d["overlayPid"] = _overlay != null && !_overlay.HasExited ? (object)(long)_overlay.Id : null;
641
- d["ok"] = true;
642
- d["uia"] = true;
643
- return d;
644
- }
645
-
646
- /// Run cheaply before every request: if a previous overlay died while it was hiding
647
- /// the system cursors, put them back. A blank desktop cursor is worse than a stray
648
- /// visible one, so this never assumes the overlay cleaned up after itself.
649
- public void Maintenance()
650
- {
651
- try
652
- {
653
- string flag = OverlayCursorFlagPath();
654
- if (!File.Exists(flag)) return;
655
- int pid;
656
- if (!int.TryParse(File.ReadAllText(flag).Trim(), out pid)) { RestoreCursors(); return; }
657
- bool alive;
658
- try { Process.GetProcessById(pid); alive = true; } catch { alive = false; }
659
- if (!alive) { RestoreCursors(); try { File.Delete(flag); } catch { } }
660
- }
661
- catch { }
662
- }
663
-
664
- static string OverlayCursorFlagPath()
665
- {
666
- return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "cursor-hidden.flag");
667
- }
668
-
669
- static void RestoreCursors()
670
- {
671
- try { SystemParametersInfo(0x0057, 0, IntPtr.Zero, 0); } catch { }
672
- }
673
-
674
- public Dictionary<string, object> RestoreCursor(Dictionary<string, object> p)
675
- {
676
- RestoreCursors();
677
- try { File.Delete(OverlayCursorFlagPath()); } catch { }
678
- Dictionary<string, object> r = new Dictionary<string, object>();
679
- r["restored"] = true;
680
- return r;
681
- }
682
-
683
- public Dictionary<string, object> Hello(Dictionary<string, object> p)
684
- {
685
- Dictionary<string, object> d = new Dictionary<string, object>();
686
- d["ok"] = true;
687
- d["name"] = "dsh-computer-use";
688
- d["protocolVersion"] = ProtocolVersion;
689
- d["platform"] = PlatformName;
690
- List<object> caps = new List<object>();
691
- foreach (string c in new string[] {
692
- "window2", "accessibility_tree", "multi_screenshot", "wgc_capture",
693
- "virtual_cursor", "audio_recording", "turn_interrupt", "app_approval",
694
- "screenshot_ids", "budgeted_capture" }) caps.Add(c);
695
- d["capabilities"] = caps;
696
- return d;
697
- }
698
-
699
- public Dictionary<string, object> Close(Dictionary<string, object> p)
700
- {
701
- Program.RequestedExitCode = 0;
702
- Program.CloseRequested = true;
703
- Dictionary<string, object> d = new Dictionary<string, object>();
704
- d["closed"] = true;
705
- return d;
706
- }
707
-
708
- public Dictionary<string, object> ListWindows(Dictionary<string, object> p)
709
- {
710
- List<object> outList = new List<object>();
711
- List<IntPtr> hs = EnumerateWindows(true);
712
- for (int i = 0; i < hs.Count; i++)
713
- {
714
- uint pid;
715
- GetWindowThreadProcessId(hs[i], out pid);
716
- outList.Add(WindowDict(hs[i], pid, i));
717
- }
718
- Dictionary<string, object> d = new Dictionary<string, object>();
719
- d["windows"] = outList;
720
- d["count"] = outList.Count;
721
- return d;
722
- }
723
-
724
- public Dictionary<string, object> ListApps(Dictionary<string, object> p)
725
- {
726
- Dictionary<string, List<object>> grouped = new Dictionary<string, List<object>>();
727
- Dictionary<string, Dictionary<string, object>> appInfo = new Dictionary<string, Dictionary<string, object>>();
728
- List<IntPtr> hs = EnumerateWindows(true);
729
- for (int i = 0; i < hs.Count; i++)
730
- {
731
- uint pid;
732
- GetWindowThreadProcessId(hs[i], out pid);
733
- Dictionary<string, object> app = AppOf(pid);
734
- string key = Convert.ToString(app["id"]);
735
- if (!grouped.ContainsKey(key)) { grouped[key] = new List<object>(); appInfo[key] = app; }
736
- grouped[key].Add(WindowDict(hs[i], pid, i));
737
- }
738
- List<object> apps = new List<object>();
739
- foreach (KeyValuePair<string, List<object>> kv in grouped)
740
- {
741
- Dictionary<string, object> a = appInfo[kv.Key];
742
- a["isRunning"] = true;
743
- try
744
- {
745
- IntPtr sample = IntPtr.Zero;
746
- List<IntPtr> windowsOfApp = new List<IntPtr>();
747
- for (int w = 0; w < hs.Count; w++)
748
- {
749
- uint wpid;
750
- GetWindowThreadProcessId(hs[w], out wpid);
751
- if (Convert.ToString(AppOf(wpid)["id"]) == kv.Key) { sample = hs[w]; break; }
752
- }
753
- if (sample != IntPtr.Zero)
754
- {
755
- uint spid;
756
- GetWindowThreadProcessId(sample, out spid);
757
- a["lastUsedDate"] = Process.GetProcessById((int)spid).StartTime.ToUniversalTime().ToString("o");
758
- }
759
- }
760
- catch { }
761
- a["windows"] = kv.Value;
762
- apps.Add(a);
763
- }
764
- Dictionary<string, object> d = new Dictionary<string, object>();
765
- d["apps"] = apps;
766
- d["count"] = apps.Count;
767
- return d;
768
- }
769
-
770
- public Dictionary<string, object> GetWindow(Dictionary<string, object> p)
771
- {
772
- IntPtr h = ResolveWindow(p);
773
- uint pid;
774
- GetWindowThreadProcessId(h, out pid);
775
- Dictionary<string, object> d = new Dictionary<string, object>();
776
- d["window"] = WindowDict(h, pid, 0);
777
- return d;
778
- }
779
-
780
- public Dictionary<string, object> ActivateWindow(Dictionary<string, object> p)
781
- {
782
- IntPtr h = ResolveWindow(p);
783
- Activate(h);
784
- Dictionary<string, object> d = new Dictionary<string, object>();
785
- d["activated"] = true;
786
- d["window"] = WindowDict(h, PidOf(h), 0);
787
- return d;
788
- }
789
-
790
- static uint PidOf(IntPtr h) { uint pid; GetWindowThreadProcessId(h, out pid); return pid; }
791
-
792
- /// Bring a window to the foreground and PROVE it got there. Input is never
793
- /// injected into a window that is not foreground: SendInput targets whatever
794
- /// holds focus, so an unproven activation would type into the user's window.
795
- static void Activate(IntPtr h)
796
- {
797
- if (!EnsureForeground(h))
798
- throw new ToolError("focus_failed", "could not bring window " + h.ToInt64() + " to the foreground, so the input was refused instead of landing in another window. " + DescribeForeground());
799
- }
800
-
801
- static bool EnsureForeground(IntPtr h)
802
- {
803
- if (!IsWindow(h)) throw new ToolError("window_gone", "window " + h.ToInt64() + " no longer exists; call list_windows again");
804
- if (!IsWindowEnabled(h)) throw new ToolError("window_disabled", "window " + h.ToInt64() + " is disabled (a modal dialog may be blocking it)");
805
- if (IsIconic(h)) { ShowWindow(h, 9); System.Threading.Thread.Sleep(180); }
806
- if (GetForegroundWindow() == h) return true;
807
-
808
- uint fgThread = GetWindowThreadProcessId(GetForegroundWindow(), IntPtr.Zero);
809
- uint thisThread = GetCurrentThreadId();
810
- bool attached = false;
811
- try
812
- {
813
- if (fgThread != 0 && fgThread != thisThread) attached = AttachThreadInput(thisThread, fgThread, true);
814
- ShowWindow(h, 5); // SW_SHOW
815
- BringWindowToTop(h);
816
- SetForegroundWindow(h);
817
- SetFocus(h);
818
- }
819
- finally { if (attached) AttachThreadInput(thisThread, fgThread, false); }
820
-
821
- if (WaitForeground(h, 10)) return true;
822
-
823
- // Windows refuses foreground changes from a background process. The
824
- // documented-looking escape hatches, in order of intrusiveness:
825
- // 1. SwitchToThisWindow (alt-tab semantics, no key injection)
826
- // 2. a bare Alt press, which lifts the foreground lock for this call
827
- try { SwitchToThisWindow(h, true); } catch { }
828
- if (WaitForeground(h, 10)) return true;
829
-
830
- keybd_event(0x12, 0, 0, UIntPtr.Zero); // VK_MENU down
831
- keybd_event(0x12, 0, 0x0002, UIntPtr.Zero); // VK_MENU up
832
- SetForegroundWindow(h);
833
- if (WaitForeground(h, 12)) return true;
834
- return false;
835
- }
836
-
837
- /// Name the process that currently owns focus: the usual reason a
838
- /// background host cannot steal it is an always-on-top utility or an
839
- /// elevated window.
840
- static string DescribeForeground()
841
- {
842
- IntPtr fg = GetForegroundWindow();
843
- if (fg == IntPtr.Zero) return "No window currently holds the foreground.";
844
- uint pid;
845
- GetWindowThreadProcessId(fg, out pid);
846
- string proc = "unknown";
847
- try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
848
- string title = TitleOf(fg);
849
- string cls = ClassOf(fg);
850
- return "Foreground is held by " + proc + " (pid " + pid + ", class " + cls + (title.Length > 0 ? ", title \"" + Trim(title, 60) + "\"" : "") + "). Bring the target window forward yourself, or close/pause the utility that keeps stealing focus.";
851
- }
852
-
853
- static bool WaitForeground(IntPtr h, int attempts)
854
- {
855
- for (int i = 0; i < attempts; i++)
856
- {
857
- if (GetForegroundWindow() == h) { System.Threading.Thread.Sleep(70); return true; }
858
- System.Threading.Thread.Sleep(40);
859
- }
860
- return GetForegroundWindow() == h;
861
- }
862
-
863
- static void Slow() { System.Threading.Thread.Sleep(120); }
864
-
865
- public Dictionary<string, object> LaunchApp(Dictionary<string, object> p)
866
- {
867
- string app = Json.GetString(p, "app");
868
- if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "launch_app requires 'app'");
869
- // Resolution order matches the API contract: an explicit .exe path first,
870
- // then a packaged-app id (AUMID) through shell:AppsFolder, then a shell URI
871
- // such as ms-settings:. A bare unknown string is tried verbatim last.
872
- string resolved = app;
873
- if (!File.Exists(app) && app.IndexOf('\\') < 0 && !app.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) && app.IndexOf(':') < 0)
874
- resolved = "shell:AppsFolder\\" + app;
875
- ProcessStartInfo psi = new ProcessStartInfo();
876
- psi.FileName = resolved;
877
- psi.UseShellExecute = true;
878
- List<object> args = Json.Get(p, "args") as List<object>;
879
- if (args != null && args.Count > 0)
880
- {
881
- StringBuilder argLine = new StringBuilder();
882
- for (int i = 0; i < args.Count; i++)
883
- {
884
- if (i > 0) argLine.Append(' ');
885
- string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture) ?? "";
886
- // A quote or newline would let a caller smuggle extra arguments past the
887
- // single quoting rule below, so refuse instead of guessing.
888
- if (a.IndexOf('"') >= 0 || a.IndexOf('\n') >= 0 || a.IndexOf('\r') >= 0)
889
- throw new ToolError("bad_request", "launch_app argument " + i + " contains a quote or newline; pass a single plain argument");
890
- if (a.Length == 0 || a.IndexOf(' ') >= 0 || a.IndexOf('\t') >= 0)
891
- argLine.Append('"').Append(a).Append('"');
892
- else argLine.Append(a);
893
- }
894
- psi.Arguments = argLine.ToString();
895
- }
896
- Process proc = Process.Start(psi);
897
- System.Threading.Thread.Sleep(900);
898
- Dictionary<string, object> d = new Dictionary<string, object>();
899
- d["launched"] = true;
900
- d["app"] = app;
901
- try { d["pid"] = (long)proc.Id; } catch { }
902
- return d;
903
- }
904
-
905
- IntPtr ResolveWindow(Dictionary<string, object> p)
906
- {
907
- Dictionary<string, object> w = Json.AsObject(Json.Get(p, "window"));
908
- long id = Json.GetLong(w, "id", 0);
909
- if (id == 0) id = Json.GetLong(p, "windowId", 0);
910
- if (id == 0) throw new ToolError("bad_request", "a 'window' object with a numeric 'id' is required (use list_windows / list_apps)");
911
- IntPtr h = new IntPtr(id);
912
- if (!IsWindow(h)) throw new ToolError("window_gone", "window " + id + " no longer exists; call list_windows again");
913
- return h;
914
- }
915
-
916
- // ---- observation ----
917
- public Dictionary<string, object> GetWindowState(Dictionary<string, object> p)
918
- {
919
- IntPtr h = ResolveWindow(p);
920
- AssertAppApproved(h);
921
- bool includeText = Json.GetBool(p, "include_text", false);
922
- bool includeShot = Json.GetBool(p, "include_screenshot", false);
923
- long maxElements = Json.GetLong(p, "maxElements", 260);
924
- long ttl = Json.GetLong(p, "ttlMs", _snapshotTtlMs);
925
-
926
- uint pid;
927
- GetWindowThreadProcessId(h, out pid);
928
- Dictionary<string, object> result = new Dictionary<string, object>();
929
- result["window"] = WindowDict(h, pid, 0);
930
-
931
- // Codex parity: accessibility is null unless include_text was asked for.
932
- result["accessibility"] = includeText ? (object)BuildAccessibility(h, maxElements, (int)ttl) : null;
933
- bool blank;
934
- result["screenshots"] = includeShot ? (object)CaptureScreenshots(h, out blank) : new List<object>();
935
- return result;
936
- }
937
-
938
- Dictionary<string, object> BuildAccessibility(IntPtr h, long maxElements, int ttlMs)
939
- {
940
- Snapshot snap = new Snapshot();
941
- snap.WindowId = h.ToInt64();
942
- snap.At = DateTime.UtcNow;
943
-
944
- StringBuilder tree = new StringBuilder();
945
- int index = 0;
946
- AutomationElement root = null;
947
- try { root = AutomationElement.FromHandle(h); }
948
- catch (Exception ex) { throw new ToolError("uia_unavailable", "UI Automation cannot read this window: " + ex.Message); }
949
-
950
- string focused = null;
951
- Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
952
-
953
- if (index == 0)
954
- {
955
- // Electron/Canvas apps may not expose a tree until accessibility is requested;
956
- // retry once after a short delay.
957
- System.Threading.Thread.Sleep(180);
958
- try { root = AutomationElement.FromHandle(h); } catch { }
959
- Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
960
- }
961
-
962
- lock (_sync) { _snapshots[snap.WindowId] = snap; }
963
-
964
- Dictionary<string, object> state = new Dictionary<string, object>();
965
- state["tree"] = tree.ToString();
966
- state["elementCount"] = (long)index;
967
- state["elements"] = snap.Summary;
968
-
969
- // Structured context Codex exposes beside the tree. Each is best-effort:
970
- // an app that does not implement the pattern simply omits the field.
971
- try
972
- {
973
- AutomationElement doc = root.FindFirst(TreeScope.Descendants,
974
- new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Document));
975
- if (doc != null)
976
- {
977
- object dp;
978
- if (doc.TryGetCurrentPattern(TextPattern.Pattern, out dp))
979
- {
980
- string body = ((TextPattern)dp).DocumentRange.GetText(8000);
981
- if (!string.IsNullOrEmpty(body)) state["document_text"] = body;
982
- }
983
- }
984
- object focusPattern;
985
- AutomationElement focusedElement = AutomationElement.FocusedElement;
986
- if (focusedElement != null && focusedElement.TryGetCurrentPattern(TextPattern.Pattern, out focusPattern))
987
- {
988
- System.Windows.Automation.Text.TextPatternRange[] ranges = ((TextPattern)focusPattern).GetSelection();
989
- if (ranges != null && ranges.Length > 0)
990
- {
991
- string selected = ranges[0].GetText(4000);
992
- if (!string.IsNullOrEmpty(selected)) state["selected_text"] = selected;
993
- }
994
- }
995
- Condition selectable = new PropertyCondition(AutomationElement.IsSelectionItemPatternAvailableProperty, true);
996
- AutomationElementCollection selectedItems = root.FindAll(TreeScope.Descendants, selectable);
997
- List<object> selectedLines = new List<object>();
998
- for (int i = 0; i < selectedItems.Count && selectedLines.Count < 20; i++)
999
- {
1000
- try
1001
- {
1002
- object sp;
1003
- if (selectedItems[i].TryGetCurrentPattern(SelectionItemPattern.Pattern, out sp) && ((SelectionItemPattern)sp).Current.IsSelected)
1004
- selectedLines.Add(SafeString(selectedItems[i].Current.Name) + " (" + SafeRole(selectedItems[i].Current) + ")");
1005
- }
1006
- catch { }
1007
- }
1008
- if (selectedLines.Count > 0) state["selected_elements"] = selectedLines;
1009
- }
1010
- catch { }
1011
- state["snapshotAt"] = snap.At.ToString("o");
1012
- state["ttlMs"] = (long)ttlMs;
1013
- if (focused != null) state["focused_element"] = focused;
1014
- return state;
1015
- }
1016
-
1017
- void Walk(AutomationElement el, int depth, long maxElements, Snapshot snap, StringBuilder sb, ref int index, IntPtr hwnd, ref string focused)
1018
- {
1019
- if (el == null) return;
1020
- AutomationElementCollection kids;
1021
- try { kids = el.FindAll(TreeScope.Children, Condition.TrueCondition); }
1022
- catch { return; }
1023
-
1024
- for (int i = 0; i < kids.Count; i++)
1025
- {
1026
- if (index >= maxElements) return;
1027
- AutomationElement k;
1028
- try { k = kids[i]; } catch { continue; }
1029
- AutomationElement.AutomationElementInformation info;
1030
- bool offscreen;
1031
- try { info = k.Current; offscreen = info.IsOffscreen; } catch { continue; }
1032
- if (offscreen) continue;
1033
- try
1034
- {
1035
-
1036
- long idx = index++;
1037
- snap.Elements[idx] = k;
1038
-
1039
- string role = SafeRole(info);
1040
- string name = SafeString(info.Name);
1041
- System.Windows.Rect rect = System.Windows.Rect.Empty;
1042
- try { rect = info.BoundingRectangle; } catch { }
1043
- bool hasRect = !rect.IsEmpty && rect.Width > 0 && rect.Height > 0;
1044
-
1045
- Dictionary<string, object> summary = new Dictionary<string, object>();
1046
- summary["index"] = idx;
1047
- summary["role"] = role;
1048
- summary["name"] = name;
1049
- if (hasRect)
1050
- {
1051
- summary["x"] = (int)rect.X;
1052
- summary["y"] = (int)rect.Y;
1053
- summary["width"] = (int)rect.Width;
1054
- summary["height"] = (int)rect.Height;
1055
- }
1056
- snap.Summary.Add(summary);
1057
-
1058
- sb.Append('[').Append(idx).Append("] ").Append(role);
1059
- if (name.Length > 0) sb.Append(" \"").Append(Trim(name, 120)).Append('"');
1060
- if (hasRect) sb.Append(" @").Append((int)rect.X).Append(',').Append((int)rect.Y).Append(' ').Append((int)rect.Width).Append('x').Append((int)rect.Height);
1061
-
1062
- List<string> actions = ActionsOf(k);
1063
- if (actions.Count > 0) sb.Append(" [").Append(string.Join(",", actions.ToArray())).Append(']');
1064
-
1065
- StringBuilder val = new StringBuilder();
1066
- try
1067
- {
1068
- object vp;
1069
- if (k.TryGetCurrentPattern(ValuePattern.Pattern, out vp))
1070
- {
1071
- string v = ((ValuePattern)vp).Current.Value;
1072
- if (!string.IsNullOrEmpty(v)) val.Append(" value=\"").Append(Trim(v, 90)).Append('"');
1073
- }
1074
- else
1075
- {
1076
- object tp;
1077
- if (k.TryGetCurrentPattern(TextPattern.Pattern, out tp))
1078
- {
1079
- string t = ((TextPattern)tp).DocumentRange.GetText(120);
1080
- if (!string.IsNullOrEmpty(t)) val.Append(" text=\"").Append(Trim(t.Replace("\r", " ").Replace("\n", " "), 90)).Append('"');
1081
- }
1082
- }
1083
- }
1084
- catch { }
1085
- sb.Append(val.ToString());
1086
-
1087
- try { if (info.HasKeyboardFocus && focused == null) focused = "[" + idx + "] " + role + " \"" + name + "\""; }
1088
- catch { }
1089
-
1090
- sb.Append('\n');
1091
- if (depth < 18) Walk(k, depth + 1, maxElements, snap, sb, ref index, hwnd, ref focused);
1092
- }
1093
- catch (Exception) { continue; }
1094
- }
1095
- }
1096
-
1097
- static string SafeRole(AutomationElement.AutomationElementInformation info)
1098
- {
1099
- try { return info.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
1100
- catch { return "Unknown"; }
1101
- }
1102
-
1103
- static long ToLong(object v) { try { return Convert.ToInt64(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 0L; } }
1104
- static double ToDouble(object v) { try { return Convert.ToDouble(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 1.0; } }
1105
-
1106
- static string SafeString(string s) { return s == null ? "" : s.Replace("\r", " ").Replace("\n", " ").Trim(); }
1107
-
1108
- static string Trim(string s, int max) { return s.Length <= max ? s : s.Substring(0, max) + "…"; }
1109
-
1110
- static List<string> ActionsOf(AutomationElement el)
1111
- {
1112
- List<string> a = new List<string>();
1113
- object pattern;
1114
- try { if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern)) a.Add("Invoke"); } catch { }
1115
- try { if (el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern)) a.Add("Toggle"); } catch { }
1116
- try { if (el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern)) a.Add("Select"); } catch { }
1117
- try { if (el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern)) a.Add("ExpandCollapse"); } catch { }
1118
- try { if (el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern)) a.Add("Scroll"); } catch { }
1119
- try { if (el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) a.Add("SetValue"); } catch { }
1120
- return a;
1121
- }
1122
-
1123
- /// Every top-level window that belongs to the same "visual state" as the
1124
- /// target: the window itself plus owned/transient popups (menus, tooltips,
1125
- /// combo drop-downs) that a screenshot of the window alone would hide.
1126
- List<IntPtr> RelatedWindows(IntPtr target)
1127
- {
1128
- RECT main = BoundsOf(target);
1129
- uint targetPid;
1130
- GetWindowThreadProcessId(target, out targetPid);
1131
- List<IntPtr> result = new List<IntPtr>();
1132
- List<IntPtr> all = EnumerateWindows(false);
1133
- for (int i = 0; i < all.Count; i++)
1134
- {
1135
- IntPtr h = all[i];
1136
- if (h == target) continue;
1137
- if (!IsWindowVisible(h)) continue;
1138
- if (IsIconic(h)) continue;
1139
- uint pid;
1140
- GetWindowThreadProcessId(h, out pid);
1141
- if (pid != targetPid) continue;
1142
- bool owned = GetWindow(h, GW_OWNER) == target;
1143
- bool popup = (GetWindowLong(h, GWL_STYLE) & WS_POPUP) != 0;
1144
- if (!owned && !popup) continue;
1145
- RECT r = BoundsOf(h);
1146
- bool intersects = r.Left < main.Right && r.Right > main.Left && r.Top < main.Bottom && r.Bottom > main.Top;
1147
- if (!intersects) continue;
1148
- result.Add(h);
1149
- }
1150
- return result;
1151
- }
1152
-
1153
- List<object> CaptureScreenshots(IntPtr h, out bool anyBlank)
1154
- {
1155
- anyBlank = false;
1156
- if (IsIconic(h))
1157
- throw new ToolError("window_minimized", "window " + h.ToInt64() + " is minimized, so there is nothing to observe: call activate_window (or restore it) and observe again. A minimized window must never be captured, because the image would not be the window.");
1158
- if (!IsWindowVisible(h))
1159
- throw new ToolError("window_not_visible", "window " + h.ToInt64() + " is not visible; activate it before observing.");
1160
-
1161
- List<IntPtr> order = EnumerateWindows(true); // z-order, topmost first
1162
- List<object> shots = new List<object>();
1163
- List<IntPtr> targets = new List<IntPtr>();
1164
- targets.Add(h);
1165
- targets.AddRange(RelatedWindows(h));
1166
-
1167
- int total = Math.Max(1, order.Count);
1168
- for (int i = 0; i < targets.Count; i++)
1169
- {
1170
- IntPtr w = targets[i];
1171
- int zIndex = order.IndexOf(w);
1172
- if (zIndex < 0) zIndex = total - 1;
1173
- Dictionary<string, object> shot = CaptureWindow(w);
1174
- shot["zIndex"] = (long)(total - zIndex); // larger = visually above
1175
- string id = "shot-" + _shotGeneration + "-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
1176
- shot["id"] = id;
1177
- RECT r = BoundsOf(w);
1178
- lock (_sync)
1179
- {
1180
- PruneShots();
1181
- _shots[id] = new ShotRegion
1182
- {
1183
- WindowId = h.ToInt64(), OriginX = r.Left, OriginY = r.Top,
1184
- Width = (int)ToLong(shot["width"]), Height = (int)ToLong(shot["height"]),
1185
- ScaleX = ToDouble(shot["scaleX"]), ScaleY = ToDouble(shot["scaleY"]),
1186
- At = DateTime.UtcNow,
1187
- };
1188
- }
1189
- shots.Add(shot);
1190
- }
1191
- return shots;
1192
- }
1193
-
1194
- void PruneShots()
1195
- {
1196
- if (_shots.Count <= 48) return;
1197
- List<string> stale = new List<string>();
1198
- DateTime cutoff = DateTime.UtcNow.AddMilliseconds(-_shotTtlMs);
1199
- foreach (KeyValuePair<string, ShotRegion> kv in _shots)
1200
- if (kv.Value.At < cutoff) stale.Add(kv.Key);
1201
- for (int i = 0; i < stale.Count; i++) _shots.Remove(stale[i]);
1202
- }
1203
-
1204
- ShotRegion ResolveShot(IntPtr h, string screenshotId)
1205
- {
1206
- if (string.IsNullOrEmpty(screenshotId))
1207
- throw new ToolError("bad_request", "screenshotId is empty");
1208
- ShotRegion region;
1209
- lock (_sync)
1210
- {
1211
- if (!_shots.TryGetValue(screenshotId, out region))
1212
- throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is no longer cached: it must come from the latest get_window_state for this window. Take a fresh screenshot before retrying.");
1213
- }
1214
- if (region.WindowId != h.ToInt64())
1215
- throw new ToolError("screenshot_mismatch", "screenshot '" + screenshotId + "' belongs to another window; take a fresh observation for this window.");
1216
- if ((DateTime.UtcNow - region.At).TotalMilliseconds > _shotTtlMs)
1217
- throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is older than the screenshot TTL. Take a fresh screenshot before retrying.");
1218
- return region;
1219
- }
1220
-
1221
- /// Codex rejects a point that falls outside the screenshot the caller referenced
1222
- /// ("is outside screenshot bounds"), because that usually means a stale observation.
1223
- static void AssertPointInsideShot(double x, double y, ShotRegion region, string screenshotId, string what)
1224
- {
1225
- if (double.IsNaN(x) || double.IsNaN(y)) return;
1226
- if (x >= 0 && y >= 0 && x <= region.Width && y <= region.Height) return;
1227
- throw new ToolError("point_outside_screenshot",
1228
- what + " (" + x.ToString("0.##") + ", " + y.ToString("0.##") + ") is outside screenshot bounds (0,0)-(" +
1229
- region.Width + "," + region.Height + ") for '" + screenshotId + "'. Take a fresh screenshot with get_window_state before retrying.");
1230
- }
1231
-
1232
- /// Windows.Graphics.Capture through the .NET 8 helper: composed window content,
1233
- /// occlusion-proof, no DPI halo. Falls back to PrintWindow/screen when absent.
1234
- static Bitmap TryCaptureWithWgc(IntPtr hwnd)
1235
- {
1236
- try
1237
- {
1238
- string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
1239
- string exe = Path.Combine(dir, "capture", "dsh-cu-capture.exe");
1240
- if (!File.Exists(exe)) return null;
1241
- string tmp = Path.Combine(Path.GetTempPath(), "dsh-cu-wgc-" + Guid.NewGuid().ToString("N") + ".png");
1242
- try
1243
- {
1244
- long budget = Budget.RemainingMs();
1245
- int waitMs = (int)Math.Max(150, Math.Min(1200, budget - 150));
1246
- ProcessStartInfo psi = new ProcessStartInfo(exe, "--hwnd " + hwnd.ToInt64().ToString() + " --out \"" + tmp + "\" --timeout " + waitMs);
1247
- psi.UseShellExecute = false;
1248
- psi.CreateNoWindow = true;
1249
- psi.RedirectStandardOutput = true;
1250
- psi.RedirectStandardError = true;
1251
- using (Process p = Process.Start(psi))
1252
- {
1253
- if (p == null) return null;
1254
- if (!p.WaitForExit(waitMs + 900)) { try { p.Kill(); } catch { } return null; }
1255
- if (p.ExitCode != 0 || !File.Exists(tmp)) return null;
1256
- byte[] bytes = File.ReadAllBytes(tmp);
1257
- using (MemoryStream ms = new MemoryStream(bytes)) return new Bitmap(ms);
1258
- }
1259
- }
1260
- finally { try { if (File.Exists(tmp)) File.Delete(tmp); } catch { } }
1261
- }
1262
- catch { return null; }
1263
- }
1264
-
1265
- Dictionary<string, object> CaptureWindow(IntPtr h)
1266
- {
1267
- RECT r = BoundsOf(h);
1268
- int w = Math.Max(1, r.Right - r.Left);
1269
- int ht = Math.Max(1, r.Bottom - r.Top);
1270
- // Cap pathological sizes so a maximised multi-monitor window cannot blow the
1271
- // attachment byte budget.
1272
- const int cap = 2560;
1273
- double scale = 1.0;
1274
- if (w > cap || ht > cap)
1275
- {
1276
- scale = Math.Min((double)cap / w, (double)cap / ht);
1277
- w = Math.Max(1, (int)(w * scale));
1278
- ht = Math.Max(1, (int)(ht * scale));
1279
- }
1280
-
1281
- Bitmap wgc = TryCaptureWithWgc(h);
1282
- if (wgc != null)
1283
- {
1284
- Dictionary<string, object> wg = FinishShot(h, wgc, r, true, 1.0, 1.0);
1285
- return wg;
1286
- }
1287
-
1288
- Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
1289
- bool printed = false;
1290
- using (Graphics g = Graphics.FromImage(bmp))
1291
- {
1292
- IntPtr hdc = g.GetHdc();
1293
- try { printed = PrintWindow(h, hdc, 2); }
1294
- finally { g.ReleaseHdc(hdc); }
1295
- }
1296
- if (!printed || IsBlank(bmp))
1297
- {
1298
- bmp.Dispose();
1299
- bmp = CaptureFromScreen(r, w, ht);
1300
- }
1301
-
1302
- // DPI-virtualized targets render at their own (usually 96 DPI) size while
1303
- // occupying a larger physical rect, so PrintWindow leaves a black halo.
1304
- // Crop to the painted region and remember the physical/screenshot ratio so
1305
- // coordinate actions can map screenshot pixels back to screen pixels.
1306
- int fullWidth = bmp.Width, fullHeight = bmp.Height;
1307
- Rectangle painted = PaintedBounds(bmp);
1308
- if (painted.Width > 16 && painted.Height > 16 && (painted.Width < bmp.Width || painted.Height < bmp.Height))
1309
- {
1310
- Bitmap cropped = bmp.Clone(new Rectangle(0, 0, painted.Right, painted.Bottom), bmp.PixelFormat);
1311
- bmp.Dispose();
1312
- bmp = cropped;
1313
- w = bmp.Width;
1314
- ht = bmp.Height;
1315
- }
1316
- double scaleX = (double)fullWidth / Math.Max(1, bmp.Width);
1317
- double scaleY = (double)fullHeight / Math.Max(1, bmp.Height);
1318
-
1319
- byte[] png;
1320
- using (MemoryStream ms = new MemoryStream())
1321
- {
1322
- bmp.Save(ms, ImageFormat.Png);
1323
- png = ms.ToArray();
1324
- }
1325
- bmp.Dispose();
1326
-
1327
- Dictionary<string, object> d = new Dictionary<string, object>();
1328
- d["mediaType"] = "image/png";
1329
- d["base64"] = Convert.ToBase64String(png);
1330
- d["width"] = (long)w;
1331
- d["height"] = (long)ht;
1332
- d["originX"] = (long)r.Left;
1333
- d["originY"] = (long)r.Top;
1334
- d["zIndex"] = 1L;
1335
- d["printed"] = printed;
1336
- d["scaleX"] = scaleX;
1337
- d["scaleY"] = scaleY;
1338
- return d;
1339
- }
1340
-
1341
- static Dictionary<string, object> FinishShot(IntPtr h, Bitmap bmp, RECT r, bool printed, double scaleX, double scaleY)
1342
- {
1343
- int w = bmp.Width, ht = bmp.Height;
1344
- byte[] png;
1345
- using (MemoryStream ms = new MemoryStream())
1346
- {
1347
- bmp.Save(ms, ImageFormat.Png);
1348
- png = ms.ToArray();
1349
- }
1350
- bmp.Dispose();
1351
- Dictionary<string, object> d = new Dictionary<string, object>();
1352
- d["mediaType"] = "image/png";
1353
- d["base64"] = Convert.ToBase64String(png);
1354
- d["width"] = (long)w;
1355
- d["height"] = (long)ht;
1356
- d["originX"] = (long)r.Left;
1357
- d["originY"] = (long)r.Top;
1358
- d["zIndex"] = 1L;
1359
- d["printed"] = printed;
1360
- d["scaleX"] = scaleX;
1361
- d["scaleY"] = scaleY;
1362
- return d;
1363
- }
1364
-
1365
- static Bitmap CaptureFromScreen(RECT r, int w, int ht)
1366
- {
1367
- Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
1368
- using (Graphics g = Graphics.FromImage(bmp))
1369
- {
1370
- try { g.CopyFromScreen(r.Left, r.Top, 0, 0, new Size(w, ht), CopyPixelOperation.SourceCopy); }
1371
- catch { }
1372
- }
1373
- return bmp;
1374
- }
1375
-
1376
- /// Bounding box of everything that is not uniform black, i.e. the region the
1377
- /// target actually painted. Used to strip the DPI-virtualization halo.
1378
- static Rectangle PaintedBounds(Bitmap bmp)
1379
- {
1380
- int right = 0, bottom = 0;
1381
- int step = Math.Max(1, Math.Min(bmp.Width, bmp.Height) / 200);
1382
- for (int y = 0; y < bmp.Height; y += step)
1383
- {
1384
- for (int x = 0; x < bmp.Width; x += step)
1385
- {
1386
- Color c = bmp.GetPixel(x, y);
1387
- if (c.R > 8 || c.G > 8 || c.B > 8)
1388
- {
1389
- if (x > right) right = x;
1390
- if (y > bottom) bottom = y;
1391
- }
1392
- }
1393
- }
1394
- return new Rectangle(0, 0, Math.Min(bmp.Width, right + step), Math.Min(bmp.Height, bottom + step));
1395
- }
1396
-
1397
- static bool IsBlank(Bitmap bmp)
1398
- {
1399
- int samples = 0, dark = 0;
1400
- int stepX = Math.Max(1, bmp.Width / 12);
1401
- int stepY = Math.Max(1, bmp.Height / 12);
1402
- for (int y = 0; y < bmp.Height; y += stepY)
1403
- {
1404
- for (int x = 0; x < bmp.Width; x += stepX)
1405
- {
1406
- Color c = bmp.GetPixel(x, y);
1407
- samples++;
1408
- if (c.R < 6 && c.G < 6 && c.B < 6) dark++;
1409
- }
1410
- }
1411
- return samples > 0 && dark == samples;
1412
- }
1413
-
1414
- // ---- element resolution ----
1415
- AutomationElement ElementAt(IntPtr hwnd, long index)
1416
- {
1417
- Snapshot snap;
1418
- lock (_sync)
1419
- {
1420
- if (!_snapshots.TryGetValue(hwnd.ToInt64(), out snap))
1421
- throw new ToolError("stale_snapshot", "no observation for this window yet: call get_window_state with include_text=true before using element_index");
1422
- }
1423
- if ((DateTime.UtcNow - snap.At).TotalMilliseconds > _snapshotTtlMs)
1424
- throw new ToolError("stale_snapshot", "the element snapshot is older than " + _snapshotTtlMs + "ms: call get_window_state again before acting on element indexes");
1425
- AutomationElement el;
1426
- if (!snap.Elements.TryGetValue(index, out el))
1427
- throw new ToolError("bad_index", "element index " + index + " is not part of the latest observation for this window");
1428
- return el;
1429
- }
1430
-
1431
- static Dictionary<string, object> ScreenshotTarget(IntPtr h, AutomationElement el)
1432
- {
1433
- RECT obs = BoundsOf(h);
1434
- System.Windows.Rect r = el.Current.BoundingRectangle;
1435
- if (r.IsEmpty || r.Width <= 0 || r.Height <= 0)
1436
- throw new ToolError("no_geometry", "the element has no bounding rectangle; use x/y coordinates instead");
1437
- Dictionary<string, object> d = new Dictionary<string, object>();
1438
- d["screenX"] = (int)r.X + (int)(r.Width / 2);
1439
- d["screenY"] = (int)r.Y + (int)(r.Height / 2);
1440
- d["originX"] = obs.Left;
1441
- d["originY"] = obs.Top;
1442
- return d;
1443
- }
1444
-
1445
- // ---- input ----
1446
- static void SendMouse(uint flags) { SendMouseData(flags, 0); }
1447
-
1448
- static void SendMouseData(uint flags, int data)
1449
- {
1450
- INPUT inp = new INPUT();
1451
- inp.type = INPUT_MOUSE;
1452
- inp.u.mi.dx = 0; inp.u.mi.dy = 0;
1453
- inp.u.mi.mouseData = (uint)data;
1454
- inp.u.mi.dwFlags = flags;
1455
- INPUT[] arr = new INPUT[] { inp };
1456
- SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1457
- }
1458
-
1459
- static void KeyDown(ushort vk) { SendKey(vk, 0, 0); }
1460
- static void KeyUp(ushort vk) { SendKey(vk, 0, KEYEVENTF_KEYUP); }
1461
-
1462
- static void SendKey(ushort vk, ushort scan, uint flags)
1463
- {
1464
- INPUT inp = new INPUT();
1465
- inp.type = INPUT_KEYBOARD;
1466
- inp.u.ki.wVk = vk;
1467
- inp.u.ki.wScan = scan;
1468
- inp.u.ki.dwFlags = flags;
1469
- INPUT[] arr = new INPUT[] { inp };
1470
- SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1471
- }
1472
-
1473
- static void SendUnicode(char ch, bool up)
1474
- {
1475
- INPUT inp = new INPUT();
1476
- inp.type = INPUT_KEYBOARD;
1477
- inp.u.ki.wVk = 0;
1478
- inp.u.ki.wScan = ch;
1479
- inp.u.ki.dwFlags = KEYEVENTF_UNICODE | (up ? KEYEVENTF_KEYUP : 0);
1480
- INPUT[] arr = new INPUT[] { inp };
1481
- SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1482
- }
1483
-
1484
- void ClickAt(int screenX, int screenY, int count, string button, Dictionary<string, object> p)
1485
- {
1486
- SetCursorPos(screenX, screenY);
1487
- System.Threading.Thread.Sleep(35);
1488
- for (int i = 0; i < Math.Max(1, count); i++)
1489
- {
1490
- CheckCancel(p);
1491
- if (i > 0) System.Threading.Thread.Sleep(60);
1492
- uint down = MOUSEEVENTF_LEFTDOWN, up = MOUSEEVENTF_LEFTUP;
1493
- if (button == "r") { down = MOUSEEVENTF_RIGHTDOWN; up = MOUSEEVENTF_RIGHTUP; }
1494
- else if (button == "m") { down = MOUSEEVENTF_MIDDLEDOWN; up = MOUSEEVENTF_MIDDLEUP; }
1495
- SendMouse(down);
1496
- try { System.Threading.Thread.Sleep(22); }
1497
- finally { SendMouse(up); }
1498
- }
1499
- }
1500
-
1501
- public Dictionary<string, object> Click(Dictionary<string, object> p)
1502
- {
1503
- CheckCancel(p);
1504
- AssertUnlocked();
1505
- EnsureOverlay(8000);
1506
- IntPtr h = ResolveWindow(p);
1507
- AssertAppApproved(h);
1508
- long elementIndex = Json.GetLong(p, "element_index", -1);
1509
- string button = NormalizeButton(Json.GetString(p, "mouse_button"));
1510
- int count = (int)Json.GetLong(p, "click_count", 1);
1511
- int sx, sy;
1512
-
1513
- if (elementIndex >= 0)
1514
- {
1515
- AutomationElement el = ElementAt(h, elementIndex);
1516
- Dictionary<string, object> t = ScreenshotTarget(h, el);
1517
- sx = (int)t["screenX"]; sy = (int)t["screenY"];
1518
- // Prefer the semantic invoke when the element exposes one and we are
1519
- // doing a plain left single click: it survives occlusion and does not
1520
- // depend on pixel geometry.
1521
- if (count == 1 && button == "l")
1522
- {
1523
- object pattern;
1524
- try
1525
- {
1526
- if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1527
- {
1528
- ((InvokePattern)pattern).Invoke();
1529
- Dictionary<string, object> r2 = new Dictionary<string, object>();
1530
- r2["clicked"] = "invoke"; r2["element_index"] = elementIndex;
1531
- return r2;
1532
- }
1533
- }
1534
- catch (Exception) { }
1535
- }
1536
- }
1537
- else
1538
- {
1539
- double x = Json.GetDouble(p, "x", double.NaN);
1540
- double y = Json.GetDouble(p, "y", double.NaN);
1541
- if (double.IsNaN(x) || double.IsNaN(y)) throw new ToolError("bad_request", "click needs either 'element_index' or both 'x' and 'y' (window-relative pixels)");
1542
- string screenshotId = Json.GetString(p, "screenshotId");
1543
- int originX, originY;
1544
- double coordScaleX = 1.0, coordScaleY = 1.0;
1545
- if (!string.IsNullOrEmpty(screenshotId))
1546
- {
1547
- ShotRegion region = ResolveShot(h, screenshotId);
1548
- AssertPointInsideShot(x, y, region, screenshotId, "click");
1549
- originX = region.OriginX; originY = region.OriginY;
1550
- coordScaleX = region.ScaleX; coordScaleY = region.ScaleY;
1551
- }
1552
- else
1553
- {
1554
- RECT bounds = BoundsOf(h);
1555
- originX = bounds.Left; originY = bounds.Top;
1556
- }
1557
- sx = originX + (int)Math.Round(x * coordScaleX);
1558
- sy = originY + (int)Math.Round(y * coordScaleY);
1559
- }
1560
-
1561
- Activate(h); // pointer clicks need a real foreground window
1562
- MarkCursor(sx, sy);
1563
- CheckCancel(p);
1564
- ClickAt(sx, sy, count, button, p);
1565
- Dictionary<string, object> r = new Dictionary<string, object>();
1566
- r["clicked"] = "pointer";
1567
- r["screenX"] = (long)sx;
1568
- r["screenY"] = (long)sy;
1569
- r["clickCount"] = (long)count;
1570
- return r;
1571
- }
1572
-
1573
- static string NormalizeButton(string b)
1574
- {
1575
- if (string.IsNullOrEmpty(b)) return "l";
1576
- string s = b.Trim().ToLowerInvariant();
1577
- if (s == "right" || s == "r") return "r";
1578
- if (s == "middle" || s == "m") return "m";
1579
- return "l";
1580
- }
1581
-
1582
- public Dictionary<string, object> PressKey(Dictionary<string, object> p)
1583
- {
1584
- CheckCancel(p);
1585
- AssertUnlocked();
1586
- EnsureOverlay(8000);
1587
- IntPtr h = ResolveWindow(p);
1588
- AssertAppApproved(h);
1589
- string key = Json.GetString(p, "key");
1590
- if (string.IsNullOrEmpty(key)) throw new ToolError("bad_request", "press_key requires 'key'");
1591
- Activate(h);
1592
- Chord(key, p);
1593
- Dictionary<string, object> r = new Dictionary<string, object>();
1594
- r["pressed"] = key;
1595
- return r;
1596
- }
1597
-
1598
- static readonly Dictionary<string, ushort> KeyMap = BuildKeyMap();
1599
-
1600
- static Dictionary<string, ushort> BuildKeyMap()
1601
- {
1602
- Dictionary<string, ushort> m = new Dictionary<string, ushort>(StringComparer.OrdinalIgnoreCase);
1603
- for (char c = 'a'; c <= 'z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'a'));
1604
- for (char c = 'A'; c <= 'Z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'A'));
1605
- for (char c = '0'; c <= '9'; c++) m[c.ToString()] = (ushort)(0x30 + (c - '0'));
1606
- m["return"] = 0x0D; m["enter"] = 0x0D; m["kp_enter"] = 0x0D;
1607
- m["tab"] = 0x09; m["space"] = 0x20; m["backspace"] = 0x08; m["escape"] = 0x1B; m["esc"] = 0x1B;
1608
- m["delete"] = 0x2E; m["insert"] = 0x2D; m["home"] = 0x24; m["end"] = 0x23;
1609
- m["page_up"] = 0x21; m["pageup"] = 0x21; m["prior"] = 0x21; m["page_down"] = 0x22; m["pagedown"] = 0x22; m["next"] = 0x22;
1610
- m["left"] = 0x25; m["up"] = 0x26; m["right"] = 0x27; m["down"] = 0x28;
1611
- m["shift_l"] = 0xA0; m["shift"] = 0xA0; m["shift_r"] = 0xA1;
1612
- m["control_l"] = 0xA2; m["control"] = 0xA2; m["ctrl"] = 0xA2; m["control_r"] = 0xA3;
1613
- m["alt_l"] = 0xA4; m["alt"] = 0xA4; m["alt_r"] = 0xA5;
1614
- m["super_l"] = 0x5B; m["win"] = 0x5B; m["meta"] = 0x5B; m["lwin"] = 0x5B; m["super"] = 0x5B;
1615
- m["super_r"] = 0x5C; m["rwin"] = 0x5C;
1616
- m["caps_lock"] = 0x14; m["num_lock"] = 0x90; m["scroll_lock"] = 0x91;
1617
- m["menu"] = 0x5D; m["apps"] = 0x5D; m["print"] = 0x2C; m["pause"] = 0x13;
1618
- m["period"] = 0xBE; m["greater"] = 0xBE; m["comma"] = 0xBC; m["less"] = 0xBC;
1619
- m["slash"] = 0xBF; m["question"] = 0xBF; m["backslash"] = 0xDC; m["bar"] = 0xDC;
1620
- m["semicolon"] = 0xBA; m["colon"] = 0xBA; m["apostrophe"] = 0xDE; m["quote"] = 0xDE;
1621
- m["minus"] = 0xBD; m["underscore"] = 0xBD; m["equal"] = 0xBB; m["plus"] = 0xBB;
1622
- m["bracketleft"] = 0xDB; m["bracketright"] = 0xDD; m["grave"] = 0xC0; m["asciitilde"] = 0xC0;
1623
- for (int i = 1; i <= 24; i++) { m["f" + i] = (ushort)(0x6F + i); }
1624
- // Numpad: both X keysym style (KP_0) and the aliases the API documents.
1625
- for (int i = 0; i <= 9; i++)
1626
- {
1627
- m["kp_" + i] = (ushort)(0x60 + i);
1628
- m["numpad_" + i] = (ushort)(0x60 + i);
1629
- }
1630
- m["kp_add"] = 0x6B; m["numpad_add"] = 0x6B; m["plus_numpad"] = 0x6B;
1631
- m["kp_subtract"] = 0x6D; m["numpad_subtract"] = 0x6D; m["minus_numpad"] = 0x6D;
1632
- m["kp_multiply"] = 0x6A; m["numpad_multiply"] = 0x6A; m["kp_asterisk"] = 0x6A;
1633
- m["kp_divide"] = 0x6F; m["numpad_divide"] = 0x6F; m["kp_slash"] = 0x6F;
1634
- m["kp_decimal"] = 0x6E; m["numpad_decimal"] = 0x6E; m["kp_separator"] = 0x6C;
1635
- m["numpad_enter"] = 0x0D;
1636
- return m;
1637
- }
1638
-
1639
- static readonly HashSet<ushort> ExtendedKeys = new HashSet<ushort>(new ushort[] {
1640
- 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x2D,0x2E,0x5B,0x5C,0x5D,0x90,0x6F,0xA3,0xA5
1641
- });
1642
-
1643
- static void SendVk(ushort vk, bool up)
1644
- {
1645
- uint flags = up ? KEYEVENTF_KEYUP : 0;
1646
- if (ExtendedKeys.Contains(vk)) flags |= KEYEVENTF_EXTENDEDKEY;
1647
- SendKey(vk, 0, flags);
1648
- }
1649
-
1650
- void Chord(string chord, Dictionary<string, object> p)
1651
- {
1652
- string[] parts = chord.Split('+');
1653
- List<ushort> keys = new List<ushort>();
1654
- foreach (string raw in parts)
1655
- {
1656
- string token = raw.Trim();
1657
- if (token.Length == 0) continue;
1658
- keys.Add(ResolveKey(token));
1659
- }
1660
- if (keys.Count == 0) throw new ToolError("bad_request", "empty key chord");
1661
- int pressed = 0;
1662
- try
1663
- {
1664
- for (int i = 0; i < keys.Count; i++)
1665
- {
1666
- CheckCancel(p);
1667
- SendVk(keys[i], false); pressed++;
1668
- System.Threading.Thread.Sleep(12);
1669
- }
1670
- }
1671
- finally { for (int i = pressed - 1; i >= 0; i--) SendVk(keys[i], true); }
1672
- }
1673
-
1674
- /// The Windows/Meta key family is hard-denied: it opens Run, Settings and
1675
- /// shell surfaces that Computer Use must never drive. Enforced in the helper
1676
- /// as well as in the plugin, so a direct protocol call cannot bypass it.
1677
- static readonly HashSet<string> DeniedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
1678
- "win", "win_l", "win_r", "lwin", "rwin", "windows", "windows_l", "windows_r",
1679
- "super", "super_l", "super_r", "meta", "meta_l", "meta_r", "cmd", "command", "os", "os_l", "os_r"
1680
- };
1681
-
1682
- static ushort ResolveKey(string token)
1683
- {
1684
- if (DeniedKeys.Contains(token))
1685
- throw new ToolError("hard_denied", "the Windows/Meta/Super key ('" + token + "') is hard-denied: it opens Run, Settings and shell surfaces that Computer Use must never drive.");
1686
- ushort vk;
1687
- if (KeyMap.TryGetValue(token, out vk)) return vk;
1688
- if (token.Length == 1)
1689
- {
1690
- char c = token[0];
1691
- short scan = VkKeyScan(c);
1692
- if (scan != -1) return (ushort)(scan & 0xFF);
1693
- return (ushort)char.ToUpperInvariant(c);
1694
- }
1695
- throw new ToolError("bad_key", "unknown key name '" + token + "'");
1696
- }
1697
-
1698
- [DllImport("user32.dll")] static extern short VkKeyScan(char ch);
1699
-
1700
- public Dictionary<string, object> TypeText(Dictionary<string, object> p)
1701
- {
1702
- CheckCancel(p);
1703
- AssertUnlocked();
1704
- EnsureOverlay(8000);
1705
- IntPtr h = ResolveWindow(p);
1706
- AssertAppApproved(h);
1707
- string text = Json.GetString(p, "text");
1708
- if (text == null) throw new ToolError("bad_request", "type_text requires 'text'");
1709
- Activate(h);
1710
- for (int i = 0; i < text.Length; i++)
1711
- {
1712
- CheckCancel(p);
1713
- char c = text[i];
1714
- if (c == '\n') { SendVk(0x0D, false); SendVk(0x0D, true); continue; }
1715
- if (c == '\t') { SendVk(0x09, false); SendVk(0x09, true); continue; }
1716
- if (c == '\r') continue;
1717
- SendUnicode(c, false);
1718
- SendUnicode(c, true);
1719
- if ((i & 15) == 0) System.Threading.Thread.Sleep(4);
1720
- }
1721
- Dictionary<string, object> r = new Dictionary<string, object>();
1722
- r["typed"] = (long)text.Length;
1723
- return r;
1724
- }
1725
-
1726
- public Dictionary<string, object> Scroll(Dictionary<string, object> p)
1727
- {
1728
- CheckCancel(p);
1729
- AssertUnlocked();
1730
- EnsureOverlay(8000);
1731
- IntPtr h = ResolveWindow(p);
1732
- AssertAppApproved(h);
1733
- // Validation first: an invalid call must not activate a window or move the pointer.
1734
- double scrollY = Json.GetDouble(p, "scrollY", 0);
1735
- double scrollX = Json.GetDouble(p, "scrollX", 0);
1736
- double x = Json.GetDouble(p, "x", double.NaN);
1737
- double y = Json.GetDouble(p, "y", double.NaN);
1738
- if (double.IsNaN(x) || double.IsNaN(y))
1739
- throw new ToolError("bad_request", "scroll requires finite 'x' and 'y' (the point to scroll from, in window/screenshot coordinates); click a pane first if it needs focus");
1740
- if (Math.Abs(scrollX) < 0.0001 && Math.Abs(scrollY) < 0.0001)
1741
- throw new ToolError("bad_request", "scroll requires a non-zero scrollX or scrollY");
1742
- Activate(h);
1743
- int sx, sy;
1744
- string scrollShot = Json.GetString(p, "screenshotId");
1745
- int originLeft, originTop;
1746
- double scrollScaleX = 1.0, scrollScaleY = 1.0;
1747
- if (!string.IsNullOrEmpty(scrollShot))
1748
- {
1749
- ShotRegion region = ResolveShot(h, scrollShot);
1750
- AssertPointInsideShot(x, y, region, scrollShot, "scroll origin");
1751
- originLeft = region.OriginX; originTop = region.OriginY;
1752
- scrollScaleX = region.ScaleX; scrollScaleY = region.ScaleY;
1753
- }
1754
- else
1755
- {
1756
- RECT b = BoundsOf(h);
1757
- originLeft = b.Left; originTop = b.Top;
1758
- }
1759
- sx = originLeft + (int)Math.Round(x * scrollScaleX);
1760
- sy = originTop + (int)Math.Round(y * scrollScaleY);
1761
- MarkCursor(sx, sy);
1762
- SetCursorPos(sx, sy);
1763
- System.Threading.Thread.Sleep(40);
1764
- if (Math.Abs(scrollY) > 0.0001) SendMouseData(MOUSEEVENTF_WHEEL, (int)(-scrollY * 120));
1765
- if (Math.Abs(scrollX) > 0.0001) SendMouseData(MOUSEEVENTF_HWHEEL, (int)(-scrollX * 120));
1766
- Dictionary<string, object> r = new Dictionary<string, object>();
1767
- r["scrolled"] = true;
1768
- r["screenX"] = (long)sx; r["screenY"] = (long)sy;
1769
- return r;
1770
- }
1771
-
1772
- public Dictionary<string, object> Drag(Dictionary<string, object> p)
1773
- {
1774
- CheckCancel(p);
1775
- AssertUnlocked();
1776
- EnsureOverlay(8000);
1777
- IntPtr h = ResolveWindow(p);
1778
- AssertAppApproved(h);
1779
- double fromX = Json.GetDouble(p, "from_x", double.NaN);
1780
- double fromY = Json.GetDouble(p, "from_y", double.NaN);
1781
- double toX = Json.GetDouble(p, "to_x", double.NaN);
1782
- double toY = Json.GetDouble(p, "to_y", double.NaN);
1783
- if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
1784
- throw new ToolError("bad_request", "drag requires finite from_x/from_y/to_x/to_y (window-relative pixels)");
1785
- Activate(h);
1786
- int originX, originY;
1787
- double dragScaleX = 1.0, dragScaleY = 1.0;
1788
- string dragShot = Json.GetString(p, "screenshotId");
1789
- if (!string.IsNullOrEmpty(dragShot))
1790
- {
1791
- ShotRegion region = ResolveShot(h, dragShot);
1792
- AssertPointInsideShot(fromX, fromY, region, dragShot, "drag start");
1793
- AssertPointInsideShot(toX, toY, region, dragShot, "drag end");
1794
- originX = region.OriginX; originY = region.OriginY;
1795
- dragScaleX = region.ScaleX; dragScaleY = region.ScaleY;
1796
- }
1797
- else
1798
- {
1799
- RECT b = BoundsOf(h);
1800
- originX = b.Left; originY = b.Top;
1801
- }
1802
- int fx = originX + (int)Math.Round(fromX * dragScaleX);
1803
- int fy = originY + (int)Math.Round(fromY * dragScaleY);
1804
- int tx = originX + (int)Math.Round(toX * dragScaleX);
1805
- int ty = originY + (int)Math.Round(toY * dragScaleY);
1806
-
1807
- MarkCursor(fx, fy);
1808
- SetCursorPos(fx, fy);
1809
- System.Threading.Thread.Sleep(60);
1810
- CheckCancel(p);
1811
- SendMouse(MOUSEEVENTF_LEFTDOWN);
1812
- try
1813
- {
1814
- System.Threading.Thread.Sleep(80);
1815
- const int steps = 14;
1816
- for (int i = 1; i <= steps; i++)
1817
- {
1818
- CheckCancel(p);
1819
- int x = fx + (tx - fx) * i / steps;
1820
- int y = fy + (ty - fy) * i / steps;
1821
- SetCursorPos(x, y);
1822
- System.Threading.Thread.Sleep(18);
1823
- }
1824
- System.Threading.Thread.Sleep(90);
1825
- CheckCancel(p);
1826
- }
1827
- finally { SendMouse(MOUSEEVENTF_LEFTUP); }
1828
-
1829
- Dictionary<string, object> r = new Dictionary<string, object>();
1830
- r["dragged"] = true;
1831
- r["from"] = new long[] { fx, fy };
1832
- r["to"] = new long[] { tx, ty };
1833
- return r;
1834
- }
1835
-
1836
- public Dictionary<string, object> SetValue(Dictionary<string, object> p)
1837
- {
1838
- CheckCancel(p);
1839
- AssertUnlocked();
1840
- EnsureOverlay(8000);
1841
- IntPtr h = ResolveWindow(p);
1842
- AssertAppApproved(h);
1843
- long elementIndex = Json.GetLong(p, "element_index", -1);
1844
- if (elementIndex < 0) throw new ToolError("bad_request", "set_value requires 'element_index'");
1845
- string value = Json.GetString(p, "value");
1846
- if (value == null) throw new ToolError("bad_request", "set_value requires 'value'");
1847
- AutomationElement el = ElementAt(h, elementIndex);
1848
- object pattern;
1849
- if (!el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern))
1850
- throw new ToolError("pattern_unavailable", "element [" + elementIndex + "] does not expose the Value pattern; use click + type_text instead");
1851
- ((ValuePattern)pattern).SetValue(value);
1852
- Dictionary<string, object> r = new Dictionary<string, object>();
1853
- r["valueSet"] = true;
1854
- r["element_index"] = elementIndex;
1855
- return r;
1856
- }
1857
-
1858
- public Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p)
1859
- {
1860
- CheckCancel(p);
1861
- AssertUnlocked();
1862
- EnsureOverlay(8000);
1863
- IntPtr h = ResolveWindow(p);
1864
- AssertAppApproved(h);
1865
- long elementIndex = Json.GetLong(p, "element_index", -1);
1866
- string action = Json.GetString(p, "action");
1867
- if (elementIndex < 0 || string.IsNullOrEmpty(action))
1868
- throw new ToolError("bad_request", "perform_secondary_action requires 'element_index' and 'action'");
1869
- AutomationElement el = ElementAt(h, elementIndex);
1870
- string a = action.Trim().ToLowerInvariant();
1871
- object pattern;
1872
-
1873
- if ((a == "raise" || a == "focus") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1874
- { ((InvokePattern)pattern).Invoke(); }
1875
- else if (a == "raise" || a == "focus") { try { el.SetFocus(); } catch (Exception ex) { throw new ToolError("action_failed", ex.Message); } }
1876
- else if ((a == "invoke" || a == "press") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1877
- { ((InvokePattern)pattern).Invoke(); }
1878
- else if ((a == "expand" || a == "collapse") && el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern))
1879
- {
1880
- ExpandCollapsePattern ec = (ExpandCollapsePattern)pattern;
1881
- if (a == "expand") ec.Expand(); else ec.Collapse();
1882
- }
1883
- else if ((a == "toggle") && el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern))
1884
- { ((TogglePattern)pattern).Toggle(); }
1885
- else if ((a == "select" || a == "selectitem") && el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern))
1886
- { ((SelectionItemPattern)pattern).Select(); }
1887
- else if (a.StartsWith("scroll") && el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern))
1888
- {
1889
- ScrollPattern sp = (ScrollPattern)pattern;
1890
- if (a.Contains("up")) sp.ScrollVertical(ScrollAmount.SmallDecrement);
1891
- else if (a.Contains("down")) sp.ScrollVertical(ScrollAmount.SmallIncrement);
1892
- else if (a.Contains("left")) sp.ScrollHorizontal(ScrollAmount.SmallDecrement);
1893
- else if (a.Contains("right")) sp.ScrollHorizontal(ScrollAmount.SmallIncrement);
1894
- else throw new ToolError("bad_action", "unsupported scroll direction in '" + action + "'");
1895
- }
1896
- else
1897
- {
1898
- throw new ToolError("action_unavailable", "element [" + elementIndex + "] does not expose action '" + action + "'");
1899
- }
1900
-
1901
- Dictionary<string, object> r = new Dictionary<string, object>();
1902
- r["performed"] = action;
1903
- r["element_index"] = elementIndex;
1904
- return r;
1905
- }
1906
- }
1
+
2
+ using System;
3
+ using System.Collections.Generic;
4
+ using System.Diagnostics;
5
+ using System.Drawing;
6
+ using System.Drawing.Imaging;
7
+ using System.IO;
8
+ using System.Runtime.InteropServices;
9
+ using System.Text;
10
+ using System.Windows.Automation;
11
+
12
+ // ---------------------------------------------------------------------------
13
+ // Platform driver contract. Windows is the only implementation today; the
14
+ // interface exists so macOS (AX + CGEvent) and Linux (AT-SPI) drivers can be
15
+ // added without touching the DSH plugin layer or the stdio protocol.
16
+ // ---------------------------------------------------------------------------
17
+ public interface IPlatformDriver : IDisposable
18
+ {
19
+ string PlatformName { get; }
20
+ Dictionary<string, object> Ping(Dictionary<string, object> p);
21
+ Dictionary<string, object> Hello(Dictionary<string, object> p);
22
+ Dictionary<string, object> RestoreCursor(Dictionary<string, object> p);
23
+ void Maintenance();
24
+ Dictionary<string, object> Close(Dictionary<string, object> p);
25
+ Dictionary<string, object> ListWindows(Dictionary<string, object> p);
26
+ Dictionary<string, object> ListApps(Dictionary<string, object> p);
27
+ Dictionary<string, object> GetWindow(Dictionary<string, object> p);
28
+ Dictionary<string, object> LaunchApp(Dictionary<string, object> p);
29
+ Dictionary<string, object> GetWindowState(Dictionary<string, object> p);
30
+ Dictionary<string, object> Click(Dictionary<string, object> p);
31
+ Dictionary<string, object> PressKey(Dictionary<string, object> p);
32
+ Dictionary<string, object> TypeText(Dictionary<string, object> p);
33
+ Dictionary<string, object> Scroll(Dictionary<string, object> p);
34
+ Dictionary<string, object> Drag(Dictionary<string, object> p);
35
+ Dictionary<string, object> SetValue(Dictionary<string, object> p);
36
+ Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p);
37
+ Dictionary<string, object> ActivateWindow(Dictionary<string, object> p);
38
+ Dictionary<string, object> OverlayShow(Dictionary<string, object> p);
39
+ Dictionary<string, object> OverlayHide(Dictionary<string, object> p);
40
+ Dictionary<string, object> Configure(Dictionary<string, object> p);
41
+ Dictionary<string, object> ApproveApp(Dictionary<string, object> p);
42
+ Dictionary<string, object> EndTurn(Dictionary<string, object> p);
43
+ Dictionary<string, object> InvalidateState(Dictionary<string, object> p);
44
+ Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p);
45
+ Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p);
46
+ }
47
+
48
+ /// Per-request time budget. Codex sends x-oai-cua-request-budget-ms with every call
49
+ /// and the helper honours it; here the budget bounds capture waits and is reported
50
+ /// back when a request overruns so the client and the helper agree on the deadline.
51
+ public static class Budget
52
+ {
53
+ static long _deadlineTicks;
54
+ static string _cancelFile;
55
+
56
+ public static void Begin(long ms, string cancelFile)
57
+ {
58
+ _deadlineTicks = ms > 0 ? DateTime.UtcNow.Ticks + ms * TimeSpan.TicksPerMillisecond : 0;
59
+ _cancelFile = cancelFile;
60
+ }
61
+
62
+ public static void Check()
63
+ {
64
+ if (_cancelFile != null && File.Exists(_cancelFile))
65
+ {
66
+ string reason = "cancelled";
67
+ try { reason = File.ReadAllText(_cancelFile).Trim(); } catch { }
68
+ if (reason != "turn_ended" && reason != "timeout") reason = "cancelled";
69
+ throw new ToolError(reason, "the native request was stopped; take a fresh observation before continuing");
70
+ }
71
+ if (Expired()) throw new ToolError("timeout", "the native request exceeded its time budget");
72
+ }
73
+
74
+ public static long RemainingMs()
75
+ {
76
+ if (_deadlineTicks == 0) return long.MaxValue / 4;
77
+ return (_deadlineTicks - DateTime.UtcNow.Ticks) / TimeSpan.TicksPerMillisecond;
78
+ }
79
+
80
+ public static bool Expired()
81
+ {
82
+ return _deadlineTicks != 0 && RemainingMs() <= 0;
83
+ }
84
+ }
85
+
86
+ public class ToolError : Exception
87
+ {
88
+ public readonly string Code;
89
+ public ToolError(string code, string message) : base(message) { Code = code; }
90
+ }
91
+
92
+ // ---------------------------------------------------------------------------
93
+ // Windows driver: Win32 + UI Automation + PrintWindow.
94
+ // ---------------------------------------------------------------------------
95
+ public class WindowsDriver : IPlatformDriver
96
+ {
97
+ public string PlatformName { get { return "windows"; } }
98
+
99
+ // ---- win32 ----
100
+ delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
101
+
102
+ [DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc cb, IntPtr lParam);
103
+ [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
104
+ [DllImport("user32.dll")] static extern bool IsWindow(IntPtr hWnd);
105
+ [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowTextW(IntPtr hWnd, StringBuilder sb, int max);
106
+ [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetClassNameW(IntPtr hWnd, StringBuilder sb, int max);
107
+ [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
108
+ [DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT r);
109
+ [DllImport("user32.dll")] static extern bool GetClientRect(IntPtr hWnd, out RECT r);
110
+ [DllImport("user32.dll")] static extern bool PrintWindow(IntPtr hWnd, IntPtr hdc, uint flags);
111
+ [DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hWnd);
112
+ [DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hWnd, int cmd);
113
+ [DllImport("user32.dll")] static extern bool IsIconic(IntPtr hWnd);
114
+ [DllImport("user32.dll")] static extern bool SetCursorPos(int x, int y);
115
+ [DllImport("user32.dll")] static extern bool GetCursorPos(out POINT p);
116
+ [DllImport("user32.dll")] static extern uint SendInput(uint n, INPUT[] inputs, int size);
117
+ [DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
118
+ [DllImport("user32.dll")] static extern IntPtr SetFocus(IntPtr hWnd);
119
+ [DllImport("user32.dll")] static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach);
120
+ [DllImport("kernel32.dll")] static extern uint GetCurrentThreadId();
121
+ [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid);
122
+ [DllImport("user32.dll")] static extern bool IsWindowEnabled(IntPtr hWnd);
123
+ [DllImport("user32.dll")] static extern bool SystemParametersInfo(uint action, uint param, IntPtr pv, uint winIni);
124
+ [DllImport("user32.dll")] static extern bool SwitchToThisWindow(IntPtr hWnd, bool fAltTab);
125
+ [DllImport("user32.dll")] static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
126
+ [DllImport("user32.dll")] static extern bool BringWindowToTop(IntPtr hWnd);
127
+ [DllImport("user32.dll")] static extern IntPtr SetThreadDpiAwarenessContext(IntPtr ctx);
128
+ [DllImport("user32.dll")] static extern int GetSystemMetrics(int index);
129
+ [DllImport("user32.dll")] static extern bool SetProcessDPIAware();
130
+ [DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT r, int size);
131
+ [DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr hWnd, uint cmd);
132
+ [DllImport("user32.dll")] static extern int GetWindowLong(IntPtr hWnd, int index);
133
+
134
+ [StructLayout(LayoutKind.Sequential)] struct RECT { public int Left, Top, Right, Bottom; }
135
+ [StructLayout(LayoutKind.Sequential)] struct POINT { public int X, Y; }
136
+
137
+ [StructLayout(LayoutKind.Sequential)]
138
+ struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
139
+
140
+ [StructLayout(LayoutKind.Sequential)]
141
+ struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
142
+
143
+ [StructLayout(LayoutKind.Explicit)]
144
+ struct INPUTUNION
145
+ {
146
+ [FieldOffset(0)] public MOUSEINPUT mi;
147
+ [FieldOffset(0)] public KEYBDINPUT ki;
148
+ }
149
+
150
+ [StructLayout(LayoutKind.Sequential)]
151
+ struct INPUT { public uint type; public INPUTUNION u; }
152
+
153
+ const int INPUT_MOUSE = 0;
154
+ const int INPUT_KEYBOARD = 1;
155
+ const uint KEYEVENTF_KEYUP = 0x0002;
156
+ const uint KEYEVENTF_UNICODE = 0x0004;
157
+ const uint KEYEVENTF_EXTENDEDKEY = 0x0001;
158
+ const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
159
+ const uint MOUSEEVENTF_LEFTUP = 0x0004;
160
+ const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
161
+ const uint MOUSEEVENTF_RIGHTUP = 0x0010;
162
+ const uint MOUSEEVENTF_MIDDLEDOWN = 0x0020;
163
+ const uint MOUSEEVENTF_MIDDLEUP = 0x0040;
164
+ const uint MOUSEEVENTF_WHEEL = 0x0800;
165
+ const uint MOUSEEVENTF_HWHEEL = 0x1000;
166
+ const int DWMWA_EXTENDED_FRAME_BOUNDS = 9;
167
+
168
+ // ---- state ----
169
+ readonly object _sync = new object();
170
+ sealed class Snapshot
171
+ {
172
+ public long WindowId;
173
+ public DateTime At;
174
+ public Dictionary<long, AutomationElement> Elements = new Dictionary<long, AutomationElement>();
175
+ public Dictionary<long, string> FallbackLabels = new Dictionary<long, string>();
176
+ public List<object> Summary = new List<object>();
177
+ }
178
+
179
+ readonly Dictionary<long, Snapshot> _snapshots = new Dictionary<long, Snapshot>();
180
+ int _snapshotTtlMs = 120000;
181
+
182
+ /// A screenshot region the model was handed, so a later coordinate action can
183
+ /// be interpreted in that exact screenshot space (Codex: screenshotId).
184
+ sealed class ShotRegion
185
+ {
186
+ public long WindowId;
187
+ public int OriginX;
188
+ public int OriginY;
189
+ public int Width;
190
+ public int Height;
191
+ /// Physical pixels per screenshot pixel. 1.0 for DPI-aware targets; >1 when
192
+ /// the target is DPI-virtualized and PrintWindow returns an unscaled render.
193
+ public double ScaleX = 1.0;
194
+ public double ScaleY = 1.0;
195
+ public DateTime At;
196
+ }
197
+
198
+ // In-use indicator ("DSH 正在使用您的电脑" + whale-girl). Codex draws an animated
199
+ // border and a virtual cursor; this is the DSH equivalent, and Esc is the kill
200
+ // switch: the overlay drops a cancel file that every input path checks first.
201
+ Process _overlay;
202
+ System.Threading.Timer _cursorWatchdog;
203
+ DateTime _overlayUntil = DateTime.MinValue;
204
+ string _overlayCancelFile;
205
+ string _overlayText = "DSH 正在使用您的电脑";
206
+ string _overlayHint = "Esc 取消";
207
+ string _overlayAccent = "#3f7fd8";
208
+ string _cursorFile; // the overlay polls this to draw the virtual cursor
209
+ /// Turn token that the user interrupted with Esc. Codex persists the same
210
+ /// fact as a marker file per conversation+turn; here the token travels with
211
+ /// every input call, so the block ends exactly when the turn does.
212
+ string _interruptedTurn = null;
213
+ /// Apps the plugin approved through the DSH approval seam. Codex passes the same
214
+ /// fact per request as x-oai-cua-approved-app; here the helper is the enforcer.
215
+ readonly HashSet<string> _approvedApps = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
216
+ bool _requireAppApproval = false;
217
+ Process _audioProcess;
218
+ string _audioPath;
219
+ string _audioStopFile;
220
+
221
+ readonly Dictionary<string, ShotRegion> _shots = new Dictionary<string, ShotRegion>();
222
+ int _shotTtlMs = 120000;
223
+ int _shotSeq = 0;
224
+ readonly string _shotGeneration = Guid.NewGuid().ToString("N");
225
+
226
+ const uint GW_OWNER = 4;
227
+ const int GWL_STYLE = -16;
228
+ const int WS_POPUP = unchecked((int)0x80000000);
229
+
230
+ public WindowsDriver()
231
+ {
232
+ Maintenance(); // startup repair: a previous session may have died with cursors hidden
233
+ try { SetThreadDpiAwarenessContext(new IntPtr(-4)); } catch { }
234
+ try { SetProcessDPIAware(); } catch { }
235
+ }
236
+
237
+ public void Dispose()
238
+ {
239
+ if (_cursorWatchdog != null) { _cursorWatchdog.Dispose(); _cursorWatchdog = null; }
240
+ StopOverlay();
241
+ }
242
+
243
+ // ---- approval plumbing ----
244
+ public Dictionary<string, object> Configure(Dictionary<string, object> p)
245
+ {
246
+ _requireAppApproval = Json.GetBool(p, "requireAppApproval", _requireAppApproval);
247
+ Dictionary<string, object> r = new Dictionary<string, object>();
248
+ r["requireAppApproval"] = _requireAppApproval;
249
+ return r;
250
+ }
251
+
252
+ public Dictionary<string, object> ApproveApp(Dictionary<string, object> p)
253
+ {
254
+ string app = Json.GetString(p, "app");
255
+ if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "approve_app requires 'app'");
256
+ lock (_sync) { _approvedApps.Add(app); }
257
+ Dictionary<string, object> r = new Dictionary<string, object>();
258
+ r["approved"] = app;
259
+ r["count"] = (long)_approvedApps.Count;
260
+ return r;
261
+ }
262
+
263
+ /// Turn boundary: drop per-turn state (element snapshots, screenshot regions, the
264
+ /// Esc block, the indicator) but keep session-scoped app approvals.
265
+ public Dictionary<string, object> EndTurn(Dictionary<string, object> p)
266
+ {
267
+ string ending = Json.GetString(p, "turnToken");
268
+ if (ending != null) ClearInterrupt(ending);
269
+ lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
270
+ _interruptedTurn = null;
271
+ StopOverlay();
272
+ Dictionary<string, object> r = new Dictionary<string, object>();
273
+ r["ended"] = true;
274
+ r["approvedApps"] = (long)_approvedApps.Count;
275
+ return r;
276
+ }
277
+
278
+ public Dictionary<string, object> InvalidateState(Dictionary<string, object> p)
279
+ {
280
+ lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
281
+ StopOverlay();
282
+ return new Dictionary<string, object> { { "invalidated", true } };
283
+ }
284
+
285
+ void AssertAppApproved(IntPtr h)
286
+ {
287
+ if (!_requireAppApproval) return;
288
+ uint pid;
289
+ GetWindowThreadProcessId(h, out pid);
290
+ string id = ProcessPathOf(pid);
291
+ lock (_sync) { if (_approvedApps.Contains(id)) return; }
292
+ throw new ToolError("app_not_approved", "the Computer Use policy has not approved '" + id + "' yet. Approve the app through the DSH approval prompt before acting on its windows.");
293
+ }
294
+
295
+ // ---- audio recording (WASAPI loopback) ----
296
+ public Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p)
297
+ {
298
+ bool envEnabled = Environment.GetEnvironmentVariable("DSH_CU_AUDIO") == "1";
299
+ bool force = Json.GetBool(p, "force", false);
300
+ // Codex treats computer audio as its own app ("computer-audio") and asks for a
301
+ // separate approval before recording; the plugin passes that here.
302
+ bool audioApproved = Json.GetString(p, "approvedApp") == "computer-audio";
303
+ if (!envEnabled && !force && !audioApproved)
304
+ throw new ToolError("audio_not_enabled", "computer audio recording is not enabled. Approve it (app \"computer-audio\") or set DSH_CU_AUDIO=1.");
305
+ if (_audioProcess != null && !_audioProcess.HasExited)
306
+ throw new ToolError("audio_running", "an audio recording is already running; stop it first");
307
+
308
+ string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
309
+ string exe = Path.Combine(dir, "audio", "dsh-cu-audio.exe");
310
+ if (!File.Exists(exe)) throw new ToolError("audio_missing", "audio recorder not found at " + exe);
311
+
312
+ long maxMs = Json.GetLong(p, "maxDurationMs", 60000);
313
+ if (maxMs < 500) maxMs = 500;
314
+ if (maxMs > 300000) maxMs = 300000; // Codex caps a single recording at 5 minutes
315
+ _audioPath = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-" + Guid.NewGuid().ToString("N") + ".wav");
316
+ _audioStopFile = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-stop-" + Guid.NewGuid().ToString("N") + ".flag");
317
+ try { if (File.Exists(_audioStopFile)) File.Delete(_audioStopFile); } catch { }
318
+
319
+ ProcessStartInfo psi = new ProcessStartInfo(exe, "--out \"" + _audioPath + "\" --max-ms " + maxMs + " --stop-file \"" + _audioStopFile + "\"");
320
+ psi.UseShellExecute = false;
321
+ psi.CreateNoWindow = true;
322
+ psi.RedirectStandardOutput = true;
323
+ psi.RedirectStandardError = true;
324
+ _audioProcess = Process.Start(psi);
325
+ System.Threading.Thread.Sleep(400);
326
+ if (_audioProcess.HasExited)
327
+ throw new ToolError("audio_not_started", "computer audio recording did not start in time (the recorder exited immediately)");
328
+
329
+ Dictionary<string, object> r = new Dictionary<string, object>();
330
+ r["started"] = true;
331
+ r["filepath"] = _audioPath;
332
+ r["maxDurationMs"] = maxMs;
333
+ return r;
334
+ }
335
+
336
+ public Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p)
337
+ {
338
+ if (_audioProcess == null) throw new ToolError("audio_not_started", "no audio recording is running");
339
+ try { if (_audioStopFile != null) File.WriteAllText(_audioStopFile, "stop"); } catch { }
340
+ try { _audioProcess.WaitForExit(8000); } catch { }
341
+ try { if (_audioProcess != null && !_audioProcess.HasExited) _audioProcess.Kill(); } catch { }
342
+
343
+ string path = _audioPath;
344
+ _audioProcess = null;
345
+ _audioStopFile = null;
346
+ _audioPath = null;
347
+
348
+ byte[] audio = new byte[0];
349
+ try { if (path != null && File.Exists(path)) audio = File.ReadAllBytes(path); } catch { }
350
+
351
+ Dictionary<string, object> r = new Dictionary<string, object>();
352
+ r["filepath"] = path;
353
+ // Codex returns the raw WAV bytes plus a data URL; base64 is the JSON-safe form.
354
+ r["bytes"] = Convert.ToBase64String(audio);
355
+ r["byteLength"] = (long)audio.Length;
356
+ r["data_url"] = audio.Length > 44 ? "data:audio/wav;base64," + Convert.ToBase64String(audio) : "";
357
+ r["ok"] = audio.Length > 44;
358
+ if (audio.Length <= 44) throw new ToolError("audio_not_recorded", "computer audio recording produced no samples");
359
+ return r;
360
+ }
361
+
362
+ // ---- in-use indicator ----
363
+ static string OverlayExePath()
364
+ {
365
+ string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location);
366
+ return Path.Combine(dir ?? ".", "dsh-cu-overlay.exe");
367
+ }
368
+
369
+ public Dictionary<string, object> OverlayShow(Dictionary<string, object> p)
370
+ {
371
+ string text = Json.GetString(p, "text");
372
+ if (!string.IsNullOrEmpty(text)) _overlayText = text;
373
+ string hint = Json.GetString(p, "hint");
374
+ if (!string.IsNullOrEmpty(hint)) _overlayHint = hint;
375
+ string accent = Json.GetString(p, "accent");
376
+ if (!string.IsNullOrEmpty(accent)) _overlayAccent = accent;
377
+ EnsureOverlay(Json.GetLong(p, "ttlMs", 20000));
378
+ Dictionary<string, object> r = new Dictionary<string, object>();
379
+ r["visible"] = true;
380
+ r["cancelFile"] = _overlayCancelFile;
381
+ return r;
382
+ }
383
+
384
+ public Dictionary<string, object> OverlayHide(Dictionary<string, object> p)
385
+ {
386
+ StopOverlay();
387
+ Dictionary<string, object> r = new Dictionary<string, object>();
388
+ r["visible"] = false;
389
+ return r;
390
+ }
391
+
392
+ void EnsureOverlay(long ttlMs)
393
+ {
394
+ string exe = OverlayExePath();
395
+ if (!File.Exists(exe)) return;
396
+ string mode = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR");
397
+ if (mode != null && mode.Equals("off", StringComparison.OrdinalIgnoreCase)) { StopOverlay(); return; }
398
+ string ttlEnv = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR_TTL_MS");
399
+ long parsed;
400
+ if (ttlEnv != null && long.TryParse(ttlEnv, out parsed) && parsed >= 1000) ttlMs = parsed;
401
+ if (_overlay != null && !_overlay.HasExited && DateTime.UtcNow < _overlayUntil)
402
+ {
403
+ _overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
404
+ return;
405
+ }
406
+ StopOverlay();
407
+ _overlayCancelFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cancel-" + Guid.NewGuid().ToString("N") + ".flag");
408
+ if (_cursorFile == null) _cursorFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cursor-" + Guid.NewGuid().ToString("N").Substring(0, 8) + ".flag");
409
+ try { if (File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
410
+ StringBuilder args = new StringBuilder();
411
+ args.Append("--ttl-ms ").Append(ttlMs);
412
+ args.Append(" --cancel-file \"").Append(_overlayCancelFile).Append('"');
413
+ args.Append(" --text \"").Append(_overlayText).Append('"');
414
+ args.Append(" --hint \"").Append(_overlayHint).Append('"');
415
+ args.Append(" --owner-pid ").Append(Process.GetCurrentProcess().Id);
416
+ args.Append(" --accent \"").Append(_overlayAccent).Append('"');
417
+ if (_cursorFile != null) args.Append(" --cursor-file \"").Append(_cursorFile).Append('"');
418
+ // Suppression is the default (the virtual cursor stands in for the real one);
419
+ // DSH_CU_KEEP_CURSOR=1 keeps the real pointer visible instead.
420
+ if (Environment.GetEnvironmentVariable("DSH_CU_KEEP_CURSOR") == "1") args.Append(" --no-suppress-cursor");
421
+ try
422
+ {
423
+ ProcessStartInfo psi = new ProcessStartInfo(exe, args.ToString());
424
+ psi.UseShellExecute = false;
425
+ _overlay = Process.Start(psi);
426
+ _overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
427
+ // Codex runs a dedicated system-cursor-manager process that restores the
428
+ // pointer the moment its parent dies. This timer gives the same guarantee
429
+ // without a second exe: if the overlay vanishes with cursors hidden, the
430
+ // pointer comes back within half a second, with no request needed.
431
+ if (_cursorWatchdog == null)
432
+ _cursorWatchdog = new System.Threading.Timer(delegate { try { Maintenance(); } catch { } }, null, 500, 500);
433
+ }
434
+ catch { _overlay = null; }
435
+ }
436
+
437
+ void StopOverlay()
438
+ {
439
+ try { if (_overlay != null && !_overlay.HasExited) _overlay.Kill(); } catch { }
440
+ // Kill() skips the overlay's exit handlers, which is where it would restore the
441
+ // system cursors. Restore them from here so a terminated overlay can never leave
442
+ // the user without a pointer.
443
+ RestoreCursors();
444
+ try { File.Delete(OverlayCursorFlagPath()); } catch { }
445
+ try { if (_overlayCancelFile != null && File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
446
+ _overlay = null;
447
+ _overlayUntil = DateTime.MinValue;
448
+ }
449
+
450
+ /// Esc on the overlay drops a cancel file; honour it before any other input.
451
+ const string EscMessage = "Computer Use was stopped by the user with the physical Escape key. Stop your work, do not call further Computer Use tools in this turn, and send a final message noting that the user stopped Computer Use.";
452
+
453
+ void CheckCancel(Dictionary<string, object> p)
454
+ {
455
+ Budget.Check();
456
+ string token = p == null ? null : Json.GetString(p, "turnToken");
457
+ if (token != null && (token == _interruptedTurn || InterruptMarkerExists(token)))
458
+ throw new ToolError("turn_stopped", EscMessage);
459
+ bool overlayExited = _overlay != null && _overlay.HasExited && _overlay.ExitCode == 130;
460
+ bool cancelFile = _overlayCancelFile != null && File.Exists(_overlayCancelFile);
461
+ if (!overlayExited && !cancelFile) return;
462
+ if (cancelFile) { try { File.Delete(_overlayCancelFile); } catch { } }
463
+ _interruptedTurn = token;
464
+ PersistInterrupt(token);
465
+ StopOverlay();
466
+ Program.RequestedExitCode = 130; // Codex exits 130 when the user hits Esc
467
+ throw new ToolError("cancelled_by_user", EscMessage);
468
+ }
469
+
470
+ const int ProtocolVersion = 2;
471
+
472
+ static string InterruptMarkerPath(string token)
473
+ {
474
+ string root = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "interrupts");
475
+ StringBuilder safe = new StringBuilder();
476
+ if (token != null)
477
+ {
478
+ for (int i = 0; i < token.Length && safe.Length < 64; i++)
479
+ {
480
+ char ch = token[i];
481
+ if (char.IsLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.') safe.Append(ch);
482
+ else if (ch == ':') safe.Append('-');
483
+ }
484
+ }
485
+ if (safe.Length == 0) safe.Append("turn");
486
+ return Path.Combine(root, safe.ToString() + ".flag");
487
+ }
488
+
489
+ static void PersistInterrupt(string token)
490
+ {
491
+ try
492
+ {
493
+ string path = InterruptMarkerPath(token);
494
+ Directory.CreateDirectory(Path.GetDirectoryName(path));
495
+ File.WriteAllText(path, DateTime.UtcNow.ToString("o"));
496
+ }
497
+ catch { }
498
+ }
499
+
500
+ static bool InterruptMarkerExists(string token)
501
+ {
502
+ try { return File.Exists(InterruptMarkerPath(token)); } catch { return false; }
503
+ }
504
+
505
+ static void ClearInterrupt(string token)
506
+ {
507
+ try { File.Delete(InterruptMarkerPath(token)); } catch { }
508
+ }
509
+
510
+ /// Publish the pointer target for the virtual cursor. The overlay reads this
511
+ /// file; a missing file simply means "no cursor to draw".
512
+ void MarkCursor(int x, int y)
513
+ {
514
+ if (_cursorFile == null) return;
515
+ try { File.WriteAllText(_cursorFile, x.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + y.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture)); } catch { }
516
+ }
517
+
518
+ /// Codex reports screenLocked instead of failing with an opaque automation error.
519
+ static void AssertUnlocked()
520
+ {
521
+ IntPtr fg = GetForegroundWindow();
522
+ if (fg == IntPtr.Zero) return;
523
+ uint pid;
524
+ GetWindowThreadProcessId(fg, out pid);
525
+ string proc = "";
526
+ try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
527
+ if (proc.Equals("LockApp", StringComparison.OrdinalIgnoreCase) || proc.Equals("LogonUI", StringComparison.OrdinalIgnoreCase))
528
+ throw new ToolError("screen_locked", "the Windows desktop is locked (LockApp is in the foreground), so Computer Use cannot interact with it. Ask the user to unlock the desktop and stop trying until they do.");
529
+ }
530
+
531
+ // ---- helpers ----
532
+ static string TitleOf(IntPtr h)
533
+ {
534
+ StringBuilder sb = new StringBuilder(1024);
535
+ GetWindowTextW(h, sb, sb.Capacity);
536
+ return sb.ToString();
537
+ }
538
+
539
+ static string ClassOf(IntPtr h)
540
+ {
541
+ StringBuilder sb = new StringBuilder(256);
542
+ GetClassNameW(h, sb, sb.Capacity);
543
+ return sb.ToString();
544
+ }
545
+
546
+ static RECT BoundsOf(IntPtr h)
547
+ {
548
+ RECT r;
549
+ try
550
+ {
551
+ if (DwmGetWindowAttribute(h, DWMWA_EXTENDED_FRAME_BOUNDS, out r, Marshal.SizeOf(typeof(RECT))) == 0 && r.Right > r.Left)
552
+ return r;
553
+ }
554
+ catch { }
555
+ GetWindowRect(h, out r);
556
+ return r;
557
+ }
558
+
559
+ static Dictionary<string, object> RectDict(int x, int y, int w, int h)
560
+ {
561
+ Dictionary<string, object> d = new Dictionary<string, object>();
562
+ d["x"] = x; d["y"] = y; d["width"] = w; d["height"] = h;
563
+ return d;
564
+ }
565
+
566
+ static string ProcessPathOf(uint pid)
567
+ {
568
+ try
569
+ {
570
+ Process p = Process.GetProcessById((int)pid);
571
+ try { return p.MainModule.FileName; }
572
+ catch { return p.ProcessName; }
573
+ }
574
+ catch { return "unknown"; }
575
+ }
576
+
577
+ static string ProcessNameOf(uint pid)
578
+ {
579
+ try { return Process.GetProcessById((int)pid).ProcessName; }
580
+ catch { return "unknown"; }
581
+ }
582
+
583
+ Dictionary<string, object> AppOf(uint pid)
584
+ {
585
+ Dictionary<string, object> app = new Dictionary<string, object>();
586
+ string full = ProcessPathOf(pid);
587
+ app["id"] = full;
588
+ app["name"] = Path.GetFileNameWithoutExtension(full);
589
+ app["displayName"] = ProcessNameOf(pid);
590
+ app["pid"] = (long)pid;
591
+ try
592
+ {
593
+ Process proc = Process.GetProcessById((int)pid);
594
+ app["lastUsedDate"] = proc.StartTime.ToUniversalTime().ToString("o");
595
+ }
596
+ catch { }
597
+ return app;
598
+ }
599
+
600
+ Dictionary<string, object> WindowDict(IntPtr h, uint pid, int z)
601
+ {
602
+ RECT r = BoundsOf(h);
603
+ Dictionary<string, object> w = new Dictionary<string, object>();
604
+ w["id"] = (long)h.ToInt64();
605
+ w["title"] = TitleOf(h);
606
+ w["app"] = AppOf(pid);
607
+ w["bounds"] = RectDict(r.Left, r.Top, r.Right - r.Left, r.Bottom - r.Top);
608
+ w["zIndex"] = z;
609
+ w["className"] = ClassOf(h);
610
+ return w;
611
+ }
612
+
613
+ List<IntPtr> EnumerateWindows(bool visibleOnly)
614
+ {
615
+ List<IntPtr> list = new List<IntPtr>();
616
+ EnumWindows(delegate(IntPtr h, IntPtr lp)
617
+ {
618
+ if (!IsWindow(h)) return true;
619
+ if (visibleOnly && !IsWindowVisible(h)) return true;
620
+ uint pid;
621
+ GetWindowThreadProcessId(h, out pid);
622
+ if (pid == 0) return true;
623
+ RECT r;
624
+ GetWindowRect(h, out r);
625
+ if (r.Right - r.Left < 80 || r.Bottom - r.Top < 60) return true;
626
+ string title = TitleOf(h);
627
+ if (title.Length == 0) return true;
628
+ list.Add(h);
629
+ return true;
630
+ }, IntPtr.Zero);
631
+ return list;
632
+ }
633
+
634
+ // ---- methods ----
635
+ public Dictionary<string, object> Ping(Dictionary<string, object> p)
636
+ {
637
+ Dictionary<string, object> d = new Dictionary<string, object>();
638
+ d["platform"] = PlatformName;
639
+ d["pid"] = (long)Process.GetCurrentProcess().Id;
640
+ d["overlayPid"] = _overlay != null && !_overlay.HasExited ? (object)(long)_overlay.Id : null;
641
+ d["ok"] = true;
642
+ d["uia"] = true;
643
+ return d;
644
+ }
645
+
646
+ /// Run cheaply before every request: if a previous overlay died while it was hiding
647
+ /// the system cursors, put them back. A blank desktop cursor is worse than a stray
648
+ /// visible one, so this never assumes the overlay cleaned up after itself.
649
+ public void Maintenance()
650
+ {
651
+ try
652
+ {
653
+ string flag = OverlayCursorFlagPath();
654
+ if (!File.Exists(flag)) return;
655
+ int pid;
656
+ if (!int.TryParse(File.ReadAllText(flag).Trim(), out pid)) { RestoreCursors(); return; }
657
+ bool alive;
658
+ try { Process.GetProcessById(pid); alive = true; } catch { alive = false; }
659
+ if (!alive) { RestoreCursors(); try { File.Delete(flag); } catch { } }
660
+ }
661
+ catch { }
662
+ }
663
+
664
+ static string OverlayCursorFlagPath()
665
+ {
666
+ return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "cursor-hidden.flag");
667
+ }
668
+
669
+ static void RestoreCursors()
670
+ {
671
+ try { SystemParametersInfo(0x0057, 0, IntPtr.Zero, 0); } catch { }
672
+ }
673
+
674
+ public Dictionary<string, object> RestoreCursor(Dictionary<string, object> p)
675
+ {
676
+ RestoreCursors();
677
+ try { File.Delete(OverlayCursorFlagPath()); } catch { }
678
+ Dictionary<string, object> r = new Dictionary<string, object>();
679
+ r["restored"] = true;
680
+ return r;
681
+ }
682
+
683
+ public Dictionary<string, object> Hello(Dictionary<string, object> p)
684
+ {
685
+ Dictionary<string, object> d = new Dictionary<string, object>();
686
+ d["ok"] = true;
687
+ d["name"] = "dsh-computer-use";
688
+ d["protocolVersion"] = ProtocolVersion;
689
+ d["platform"] = PlatformName;
690
+ List<object> caps = new List<object>();
691
+ foreach (string c in new string[] {
692
+ "window2", "accessibility_tree", "multi_screenshot", "wgc_capture",
693
+ "virtual_cursor", "audio_recording", "turn_interrupt", "app_approval",
694
+ "screenshot_ids", "budgeted_capture" }) caps.Add(c);
695
+ d["capabilities"] = caps;
696
+ return d;
697
+ }
698
+
699
+ public Dictionary<string, object> Close(Dictionary<string, object> p)
700
+ {
701
+ Program.RequestedExitCode = 0;
702
+ Program.CloseRequested = true;
703
+ Dictionary<string, object> d = new Dictionary<string, object>();
704
+ d["closed"] = true;
705
+ return d;
706
+ }
707
+
708
+ public Dictionary<string, object> ListWindows(Dictionary<string, object> p)
709
+ {
710
+ List<object> outList = new List<object>();
711
+ List<IntPtr> hs = EnumerateWindows(true);
712
+ for (int i = 0; i < hs.Count; i++)
713
+ {
714
+ uint pid;
715
+ GetWindowThreadProcessId(hs[i], out pid);
716
+ outList.Add(WindowDict(hs[i], pid, i));
717
+ }
718
+ Dictionary<string, object> d = new Dictionary<string, object>();
719
+ d["windows"] = outList;
720
+ d["count"] = outList.Count;
721
+ return d;
722
+ }
723
+
724
+ public Dictionary<string, object> ListApps(Dictionary<string, object> p)
725
+ {
726
+ Dictionary<string, List<object>> grouped = new Dictionary<string, List<object>>();
727
+ Dictionary<string, Dictionary<string, object>> appInfo = new Dictionary<string, Dictionary<string, object>>();
728
+ List<IntPtr> hs = EnumerateWindows(true);
729
+ for (int i = 0; i < hs.Count; i++)
730
+ {
731
+ uint pid;
732
+ GetWindowThreadProcessId(hs[i], out pid);
733
+ Dictionary<string, object> app = AppOf(pid);
734
+ string key = Convert.ToString(app["id"]);
735
+ if (!grouped.ContainsKey(key)) { grouped[key] = new List<object>(); appInfo[key] = app; }
736
+ grouped[key].Add(WindowDict(hs[i], pid, i));
737
+ }
738
+ List<object> apps = new List<object>();
739
+ foreach (KeyValuePair<string, List<object>> kv in grouped)
740
+ {
741
+ Dictionary<string, object> a = appInfo[kv.Key];
742
+ a["isRunning"] = true;
743
+ try
744
+ {
745
+ IntPtr sample = IntPtr.Zero;
746
+ List<IntPtr> windowsOfApp = new List<IntPtr>();
747
+ for (int w = 0; w < hs.Count; w++)
748
+ {
749
+ uint wpid;
750
+ GetWindowThreadProcessId(hs[w], out wpid);
751
+ if (Convert.ToString(AppOf(wpid)["id"]) == kv.Key) { sample = hs[w]; break; }
752
+ }
753
+ if (sample != IntPtr.Zero)
754
+ {
755
+ uint spid;
756
+ GetWindowThreadProcessId(sample, out spid);
757
+ a["lastUsedDate"] = Process.GetProcessById((int)spid).StartTime.ToUniversalTime().ToString("o");
758
+ }
759
+ }
760
+ catch { }
761
+ a["windows"] = kv.Value;
762
+ apps.Add(a);
763
+ }
764
+ Dictionary<string, object> d = new Dictionary<string, object>();
765
+ d["apps"] = apps;
766
+ d["count"] = apps.Count;
767
+ return d;
768
+ }
769
+
770
+ public Dictionary<string, object> GetWindow(Dictionary<string, object> p)
771
+ {
772
+ IntPtr h = ResolveWindow(p);
773
+ uint pid;
774
+ GetWindowThreadProcessId(h, out pid);
775
+ Dictionary<string, object> d = new Dictionary<string, object>();
776
+ d["window"] = WindowDict(h, pid, 0);
777
+ return d;
778
+ }
779
+
780
+ public Dictionary<string, object> ActivateWindow(Dictionary<string, object> p)
781
+ {
782
+ IntPtr h = ResolveWindow(p);
783
+ Activate(h);
784
+ Dictionary<string, object> d = new Dictionary<string, object>();
785
+ d["activated"] = true;
786
+ d["window"] = WindowDict(h, PidOf(h), 0);
787
+ return d;
788
+ }
789
+
790
+ static uint PidOf(IntPtr h) { uint pid; GetWindowThreadProcessId(h, out pid); return pid; }
791
+
792
+ /// Bring a window to the foreground and PROVE it got there. Input is never
793
+ /// injected into a window that is not foreground: SendInput targets whatever
794
+ /// holds focus, so an unproven activation would type into the user's window.
795
+ static void Activate(IntPtr h)
796
+ {
797
+ if (!EnsureForeground(h))
798
+ throw new ToolError("focus_failed", "could not bring window " + h.ToInt64() + " to the foreground, so the input was refused instead of landing in another window. " + DescribeForeground());
799
+ }
800
+
801
+ static bool EnsureForeground(IntPtr h)
802
+ {
803
+ if (!IsWindow(h)) throw new ToolError("window_gone", "window " + h.ToInt64() + " no longer exists; call list_windows again");
804
+ if (!IsWindowEnabled(h)) throw new ToolError("window_disabled", "window " + h.ToInt64() + " is disabled (a modal dialog may be blocking it)");
805
+ if (IsIconic(h)) { ShowWindow(h, 9); System.Threading.Thread.Sleep(180); }
806
+ if (GetForegroundWindow() == h) return true;
807
+
808
+ uint fgThread = GetWindowThreadProcessId(GetForegroundWindow(), IntPtr.Zero);
809
+ uint thisThread = GetCurrentThreadId();
810
+ bool attached = false;
811
+ try
812
+ {
813
+ if (fgThread != 0 && fgThread != thisThread) attached = AttachThreadInput(thisThread, fgThread, true);
814
+ ShowWindow(h, 5); // SW_SHOW
815
+ BringWindowToTop(h);
816
+ SetForegroundWindow(h);
817
+ SetFocus(h);
818
+ }
819
+ finally { if (attached) AttachThreadInput(thisThread, fgThread, false); }
820
+
821
+ if (WaitForeground(h, 10)) return true;
822
+
823
+ // Windows refuses foreground changes from a background process. The
824
+ // documented-looking escape hatches, in order of intrusiveness:
825
+ // 1. SwitchToThisWindow (alt-tab semantics, no key injection)
826
+ // 2. a bare Alt press, which lifts the foreground lock for this call
827
+ try { SwitchToThisWindow(h, true); } catch { }
828
+ if (WaitForeground(h, 10)) return true;
829
+
830
+ keybd_event(0x12, 0, 0, UIntPtr.Zero); // VK_MENU down
831
+ keybd_event(0x12, 0, 0x0002, UIntPtr.Zero); // VK_MENU up
832
+ SetForegroundWindow(h);
833
+ if (WaitForeground(h, 12)) return true;
834
+ return false;
835
+ }
836
+
837
+ /// Name the process that currently owns focus: the usual reason a
838
+ /// background host cannot steal it is an always-on-top utility or an
839
+ /// elevated window.
840
+ static string DescribeForeground()
841
+ {
842
+ IntPtr fg = GetForegroundWindow();
843
+ if (fg == IntPtr.Zero) return "No window currently holds the foreground.";
844
+ uint pid;
845
+ GetWindowThreadProcessId(fg, out pid);
846
+ string proc = "unknown";
847
+ try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
848
+ string title = TitleOf(fg);
849
+ string cls = ClassOf(fg);
850
+ return "Foreground is held by " + proc + " (pid " + pid + ", class " + cls + (title.Length > 0 ? ", title \"" + Trim(title, 60) + "\"" : "") + "). Bring the target window forward yourself, or close/pause the utility that keeps stealing focus.";
851
+ }
852
+
853
+ static bool WaitForeground(IntPtr h, int attempts)
854
+ {
855
+ for (int i = 0; i < attempts; i++)
856
+ {
857
+ if (GetForegroundWindow() == h) { System.Threading.Thread.Sleep(70); return true; }
858
+ System.Threading.Thread.Sleep(40);
859
+ }
860
+ return GetForegroundWindow() == h;
861
+ }
862
+
863
+ static void Slow() { System.Threading.Thread.Sleep(120); }
864
+
865
+ public Dictionary<string, object> LaunchApp(Dictionary<string, object> p)
866
+ {
867
+ string app = Json.GetString(p, "app");
868
+ if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "launch_app requires 'app'");
869
+ // Resolution order matches the API contract: an explicit .exe path first,
870
+ // then a packaged-app id (AUMID) through shell:AppsFolder, then a shell URI
871
+ // such as ms-settings:. A bare unknown string is tried verbatim last.
872
+ string resolved = app;
873
+ if (!File.Exists(app) && app.IndexOf('\\') < 0 && !app.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) && app.IndexOf(':') < 0)
874
+ resolved = "shell:AppsFolder\\" + app;
875
+ ProcessStartInfo psi = new ProcessStartInfo();
876
+ psi.FileName = resolved;
877
+ psi.UseShellExecute = true;
878
+ List<object> args = Json.Get(p, "args") as List<object>;
879
+ if (args != null && args.Count > 0)
880
+ {
881
+ StringBuilder argLine = new StringBuilder();
882
+ for (int i = 0; i < args.Count; i++)
883
+ {
884
+ if (i > 0) argLine.Append(' ');
885
+ string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture) ?? "";
886
+ // A quote or newline would let a caller smuggle extra arguments past the
887
+ // single quoting rule below, so refuse instead of guessing.
888
+ if (a.IndexOf('"') >= 0 || a.IndexOf('\n') >= 0 || a.IndexOf('\r') >= 0)
889
+ throw new ToolError("bad_request", "launch_app argument " + i + " contains a quote or newline; pass a single plain argument");
890
+ if (a.Length == 0 || a.IndexOf(' ') >= 0 || a.IndexOf('\t') >= 0)
891
+ argLine.Append('"').Append(a).Append('"');
892
+ else argLine.Append(a);
893
+ }
894
+ psi.Arguments = argLine.ToString();
895
+ }
896
+ Process proc = Process.Start(psi);
897
+ System.Threading.Thread.Sleep(900);
898
+ Dictionary<string, object> d = new Dictionary<string, object>();
899
+ d["launched"] = true;
900
+ d["app"] = app;
901
+ try { d["pid"] = (long)proc.Id; } catch { }
902
+ return d;
903
+ }
904
+
905
+ IntPtr ResolveWindow(Dictionary<string, object> p)
906
+ {
907
+ Dictionary<string, object> w = Json.AsObject(Json.Get(p, "window"));
908
+ long id = Json.GetLong(w, "id", 0);
909
+ if (id == 0) id = Json.GetLong(p, "windowId", 0);
910
+ if (id == 0) throw new ToolError("bad_request", "a 'window' object with a numeric 'id' is required (use list_windows / list_apps)");
911
+ IntPtr h = new IntPtr(id);
912
+ if (!IsWindow(h)) throw new ToolError("window_gone", "window " + id + " no longer exists; call list_windows again");
913
+ return h;
914
+ }
915
+
916
+ // ---- observation ----
917
+ public Dictionary<string, object> GetWindowState(Dictionary<string, object> p)
918
+ {
919
+ IntPtr h = ResolveWindow(p);
920
+ AssertAppApproved(h);
921
+ bool includeText = Json.GetBool(p, "include_text", false);
922
+ bool includeShot = Json.GetBool(p, "include_screenshot", false);
923
+ long maxElements = Json.GetLong(p, "maxElements", 260);
924
+ long ttl = Json.GetLong(p, "ttlMs", _snapshotTtlMs);
925
+
926
+ uint pid;
927
+ GetWindowThreadProcessId(h, out pid);
928
+ Dictionary<string, object> result = new Dictionary<string, object>();
929
+ result["window"] = WindowDict(h, pid, 0);
930
+
931
+ // Codex parity: accessibility is null unless include_text was asked for.
932
+ result["accessibility"] = includeText ? (object)BuildAccessibility(h, maxElements, (int)ttl) : null;
933
+ bool blank;
934
+ result["screenshots"] = includeShot ? (object)CaptureScreenshots(h, out blank) : new List<object>();
935
+ return result;
936
+ }
937
+
938
+ Dictionary<string, object> BuildAccessibility(IntPtr h, long maxElements, int ttlMs)
939
+ {
940
+ Snapshot snap = new Snapshot();
941
+ snap.WindowId = h.ToInt64();
942
+ snap.At = DateTime.UtcNow;
943
+
944
+ StringBuilder tree = new StringBuilder();
945
+ int index = 0;
946
+ AutomationElement root = null;
947
+ try { root = AutomationElement.FromHandle(h); }
948
+ catch (Exception ex) { throw new ToolError("uia_unavailable", "UI Automation cannot read this window: " + ex.Message); }
949
+
950
+ string focused = null;
951
+ Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
952
+
953
+ if (index == 0)
954
+ {
955
+ // Electron/Canvas apps may not expose a tree until accessibility is requested;
956
+ // retry once after a short delay.
957
+ System.Threading.Thread.Sleep(180);
958
+ try { root = AutomationElement.FromHandle(h); } catch { }
959
+ Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
960
+ }
961
+
962
+ lock (_sync) { _snapshots[snap.WindowId] = snap; }
963
+
964
+ Dictionary<string, object> state = new Dictionary<string, object>();
965
+ state["tree"] = tree.ToString();
966
+ state["elementCount"] = (long)index;
967
+ state["elements"] = snap.Summary;
968
+
969
+ // Structured context Codex exposes beside the tree. Each is best-effort:
970
+ // an app that does not implement the pattern simply omits the field.
971
+ try
972
+ {
973
+ AutomationElement doc = root.FindFirst(TreeScope.Descendants,
974
+ new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Document));
975
+ if (doc != null)
976
+ {
977
+ object dp;
978
+ if (doc.TryGetCurrentPattern(TextPattern.Pattern, out dp))
979
+ {
980
+ string body = ((TextPattern)dp).DocumentRange.GetText(8000);
981
+ if (!string.IsNullOrEmpty(body)) state["document_text"] = body;
982
+ }
983
+ }
984
+ object focusPattern;
985
+ AutomationElement focusedElement = AutomationElement.FocusedElement;
986
+ if (focusedElement != null && focusedElement.TryGetCurrentPattern(TextPattern.Pattern, out focusPattern))
987
+ {
988
+ System.Windows.Automation.Text.TextPatternRange[] ranges = ((TextPattern)focusPattern).GetSelection();
989
+ if (ranges != null && ranges.Length > 0)
990
+ {
991
+ string selected = ranges[0].GetText(4000);
992
+ if (!string.IsNullOrEmpty(selected)) state["selected_text"] = selected;
993
+ }
994
+ }
995
+ Condition selectable = new PropertyCondition(AutomationElement.IsSelectionItemPatternAvailableProperty, true);
996
+ AutomationElementCollection selectedItems = root.FindAll(TreeScope.Descendants, selectable);
997
+ List<object> selectedLines = new List<object>();
998
+ for (int i = 0; i < selectedItems.Count && selectedLines.Count < 20; i++)
999
+ {
1000
+ try
1001
+ {
1002
+ object sp;
1003
+ if (selectedItems[i].TryGetCurrentPattern(SelectionItemPattern.Pattern, out sp) && ((SelectionItemPattern)sp).Current.IsSelected)
1004
+ selectedLines.Add(SafeString(selectedItems[i].Current.Name) + " (" + SafeRole(selectedItems[i].Current) + ")");
1005
+ }
1006
+ catch { }
1007
+ }
1008
+ if (selectedLines.Count > 0) state["selected_elements"] = selectedLines;
1009
+ }
1010
+ catch { }
1011
+ state["snapshotAt"] = snap.At.ToString("o");
1012
+ state["ttlMs"] = (long)ttlMs;
1013
+ if (focused != null) state["focused_element"] = focused;
1014
+ return state;
1015
+ }
1016
+
1017
+ void Walk(AutomationElement el, int depth, long maxElements, Snapshot snap, StringBuilder sb, ref int index, IntPtr hwnd, ref string focused)
1018
+ {
1019
+ if (el == null) return;
1020
+ AutomationElementCollection kids;
1021
+ try { kids = el.FindAll(TreeScope.Children, Condition.TrueCondition); }
1022
+ catch { return; }
1023
+
1024
+ for (int i = 0; i < kids.Count; i++)
1025
+ {
1026
+ if (index >= maxElements) return;
1027
+ AutomationElement k;
1028
+ try { k = kids[i]; } catch { continue; }
1029
+ AutomationElement.AutomationElementInformation info;
1030
+ bool offscreen;
1031
+ try { info = k.Current; offscreen = info.IsOffscreen; } catch { continue; }
1032
+ if (offscreen) continue;
1033
+ try
1034
+ {
1035
+
1036
+ long idx = index++;
1037
+ snap.Elements[idx] = k;
1038
+
1039
+ string role = SafeRole(info);
1040
+ string name = SafeString(info.Name);
1041
+ System.Windows.Rect rect = System.Windows.Rect.Empty;
1042
+ try { rect = info.BoundingRectangle; } catch { }
1043
+ bool hasRect = !rect.IsEmpty && rect.Width > 0 && rect.Height > 0;
1044
+
1045
+ Dictionary<string, object> summary = new Dictionary<string, object>();
1046
+ summary["index"] = idx;
1047
+ summary["role"] = role;
1048
+ summary["name"] = name;
1049
+ if (hasRect)
1050
+ {
1051
+ summary["x"] = (int)rect.X;
1052
+ summary["y"] = (int)rect.Y;
1053
+ summary["width"] = (int)rect.Width;
1054
+ summary["height"] = (int)rect.Height;
1055
+ }
1056
+ snap.Summary.Add(summary);
1057
+
1058
+ sb.Append('[').Append(idx).Append("] ").Append(role);
1059
+ if (name.Length > 0) sb.Append(" \"").Append(Trim(name, 120)).Append('"');
1060
+ if (hasRect) sb.Append(" @").Append((int)rect.X).Append(',').Append((int)rect.Y).Append(' ').Append((int)rect.Width).Append('x').Append((int)rect.Height);
1061
+
1062
+ List<string> actions = ActionsOf(k);
1063
+ if (actions.Count > 0) sb.Append(" [").Append(string.Join(",", actions.ToArray())).Append(']');
1064
+
1065
+ StringBuilder val = new StringBuilder();
1066
+ try
1067
+ {
1068
+ object vp;
1069
+ if (k.TryGetCurrentPattern(ValuePattern.Pattern, out vp))
1070
+ {
1071
+ string v = ((ValuePattern)vp).Current.Value;
1072
+ if (!string.IsNullOrEmpty(v)) val.Append(" value=\"").Append(Trim(v, 90)).Append('"');
1073
+ }
1074
+ else
1075
+ {
1076
+ object tp;
1077
+ if (k.TryGetCurrentPattern(TextPattern.Pattern, out tp))
1078
+ {
1079
+ string t = ((TextPattern)tp).DocumentRange.GetText(120);
1080
+ if (!string.IsNullOrEmpty(t)) val.Append(" text=\"").Append(Trim(t.Replace("\r", " ").Replace("\n", " "), 90)).Append('"');
1081
+ }
1082
+ }
1083
+ }
1084
+ catch { }
1085
+ sb.Append(val.ToString());
1086
+
1087
+ try { if (info.HasKeyboardFocus && focused == null) focused = "[" + idx + "] " + role + " \"" + name + "\""; }
1088
+ catch { }
1089
+
1090
+ sb.Append('\n');
1091
+ if (depth < 18) Walk(k, depth + 1, maxElements, snap, sb, ref index, hwnd, ref focused);
1092
+ }
1093
+ catch (Exception) { continue; }
1094
+ }
1095
+ }
1096
+
1097
+ static string SafeRole(AutomationElement.AutomationElementInformation info)
1098
+ {
1099
+ try { return info.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
1100
+ catch { return "Unknown"; }
1101
+ }
1102
+
1103
+ static long ToLong(object v) { try { return Convert.ToInt64(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 0L; } }
1104
+ static double ToDouble(object v) { try { return Convert.ToDouble(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 1.0; } }
1105
+
1106
+ static string SafeString(string s) { return s == null ? "" : s.Replace("\r", " ").Replace("\n", " ").Trim(); }
1107
+
1108
+ static string Trim(string s, int max) { return s.Length <= max ? s : s.Substring(0, max) + "…"; }
1109
+
1110
+ static List<string> ActionsOf(AutomationElement el)
1111
+ {
1112
+ List<string> a = new List<string>();
1113
+ object pattern;
1114
+ try { if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern)) a.Add("Invoke"); } catch { }
1115
+ try { if (el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern)) a.Add("Toggle"); } catch { }
1116
+ try { if (el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern)) a.Add("Select"); } catch { }
1117
+ try { if (el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern)) a.Add("ExpandCollapse"); } catch { }
1118
+ try { if (el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern)) a.Add("Scroll"); } catch { }
1119
+ try { if (el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) a.Add("SetValue"); } catch { }
1120
+ return a;
1121
+ }
1122
+
1123
+ /// Every top-level window that belongs to the same "visual state" as the
1124
+ /// target: the window itself plus owned/transient popups (menus, tooltips,
1125
+ /// combo drop-downs) that a screenshot of the window alone would hide.
1126
+ List<IntPtr> RelatedWindows(IntPtr target)
1127
+ {
1128
+ RECT main = BoundsOf(target);
1129
+ uint targetPid;
1130
+ GetWindowThreadProcessId(target, out targetPid);
1131
+ List<IntPtr> result = new List<IntPtr>();
1132
+ List<IntPtr> all = EnumerateWindows(false);
1133
+ for (int i = 0; i < all.Count; i++)
1134
+ {
1135
+ IntPtr h = all[i];
1136
+ if (h == target) continue;
1137
+ if (!IsWindowVisible(h)) continue;
1138
+ if (IsIconic(h)) continue;
1139
+ uint pid;
1140
+ GetWindowThreadProcessId(h, out pid);
1141
+ if (pid != targetPid) continue;
1142
+ bool owned = GetWindow(h, GW_OWNER) == target;
1143
+ bool popup = (GetWindowLong(h, GWL_STYLE) & WS_POPUP) != 0;
1144
+ if (!owned && !popup) continue;
1145
+ RECT r = BoundsOf(h);
1146
+ bool intersects = r.Left < main.Right && r.Right > main.Left && r.Top < main.Bottom && r.Bottom > main.Top;
1147
+ if (!intersects) continue;
1148
+ result.Add(h);
1149
+ }
1150
+ return result;
1151
+ }
1152
+
1153
+ List<object> CaptureScreenshots(IntPtr h, out bool anyBlank)
1154
+ {
1155
+ anyBlank = false;
1156
+ if (IsIconic(h))
1157
+ throw new ToolError("window_minimized", "window " + h.ToInt64() + " is minimized, so there is nothing to observe: call activate_window (or restore it) and observe again. A minimized window must never be captured, because the image would not be the window.");
1158
+ if (!IsWindowVisible(h))
1159
+ throw new ToolError("window_not_visible", "window " + h.ToInt64() + " is not visible; activate it before observing.");
1160
+
1161
+ List<IntPtr> order = EnumerateWindows(true); // z-order, topmost first
1162
+ List<object> shots = new List<object>();
1163
+ List<IntPtr> targets = new List<IntPtr>();
1164
+ targets.Add(h);
1165
+ targets.AddRange(RelatedWindows(h));
1166
+
1167
+ int total = Math.Max(1, order.Count);
1168
+ for (int i = 0; i < targets.Count; i++)
1169
+ {
1170
+ IntPtr w = targets[i];
1171
+ int zIndex = order.IndexOf(w);
1172
+ if (zIndex < 0) zIndex = total - 1;
1173
+ Dictionary<string, object> shot = CaptureWindow(w);
1174
+ shot["zIndex"] = (long)(total - zIndex); // larger = visually above
1175
+ string id = "shot-" + _shotGeneration + "-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
1176
+ shot["id"] = id;
1177
+ RECT r = BoundsOf(w);
1178
+ lock (_sync)
1179
+ {
1180
+ PruneShots();
1181
+ _shots[id] = new ShotRegion
1182
+ {
1183
+ WindowId = h.ToInt64(), OriginX = r.Left, OriginY = r.Top,
1184
+ Width = (int)ToLong(shot["width"]), Height = (int)ToLong(shot["height"]),
1185
+ ScaleX = ToDouble(shot["scaleX"]), ScaleY = ToDouble(shot["scaleY"]),
1186
+ At = DateTime.UtcNow,
1187
+ };
1188
+ }
1189
+ shots.Add(shot);
1190
+ }
1191
+ return shots;
1192
+ }
1193
+
1194
+ void PruneShots()
1195
+ {
1196
+ if (_shots.Count <= 48) return;
1197
+ List<string> stale = new List<string>();
1198
+ DateTime cutoff = DateTime.UtcNow.AddMilliseconds(-_shotTtlMs);
1199
+ foreach (KeyValuePair<string, ShotRegion> kv in _shots)
1200
+ if (kv.Value.At < cutoff) stale.Add(kv.Key);
1201
+ for (int i = 0; i < stale.Count; i++) _shots.Remove(stale[i]);
1202
+ }
1203
+
1204
+ ShotRegion ResolveShot(IntPtr h, string screenshotId)
1205
+ {
1206
+ if (string.IsNullOrEmpty(screenshotId))
1207
+ throw new ToolError("bad_request", "screenshotId is empty");
1208
+ ShotRegion region;
1209
+ lock (_sync)
1210
+ {
1211
+ if (!_shots.TryGetValue(screenshotId, out region))
1212
+ throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is no longer cached: it must come from the latest get_window_state for this window. Take a fresh screenshot before retrying.");
1213
+ }
1214
+ if (region.WindowId != h.ToInt64())
1215
+ throw new ToolError("screenshot_mismatch", "screenshot '" + screenshotId + "' belongs to another window; take a fresh observation for this window.");
1216
+ if ((DateTime.UtcNow - region.At).TotalMilliseconds > _shotTtlMs)
1217
+ throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is older than the screenshot TTL. Take a fresh screenshot before retrying.");
1218
+ return region;
1219
+ }
1220
+
1221
+ /// Codex rejects a point that falls outside the screenshot the caller referenced
1222
+ /// ("is outside screenshot bounds"), because that usually means a stale observation.
1223
+ static void AssertPointInsideShot(double x, double y, ShotRegion region, string screenshotId, string what)
1224
+ {
1225
+ if (double.IsNaN(x) || double.IsNaN(y)) return;
1226
+ if (x >= 0 && y >= 0 && x <= region.Width && y <= region.Height) return;
1227
+ throw new ToolError("point_outside_screenshot",
1228
+ what + " (" + x.ToString("0.##") + ", " + y.ToString("0.##") + ") is outside screenshot bounds (0,0)-(" +
1229
+ region.Width + "," + region.Height + ") for '" + screenshotId + "'. Take a fresh screenshot with get_window_state before retrying.");
1230
+ }
1231
+
1232
+ /// Windows.Graphics.Capture through the .NET 8 helper: composed window content,
1233
+ /// occlusion-proof, no DPI halo. Falls back to PrintWindow/screen when absent.
1234
+ static Bitmap TryCaptureWithWgc(IntPtr hwnd)
1235
+ {
1236
+ try
1237
+ {
1238
+ string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
1239
+ string exe = Path.Combine(dir, "capture", "dsh-cu-capture.exe");
1240
+ if (!File.Exists(exe)) return null;
1241
+ string tmp = Path.Combine(Path.GetTempPath(), "dsh-cu-wgc-" + Guid.NewGuid().ToString("N") + ".png");
1242
+ try
1243
+ {
1244
+ long budget = Budget.RemainingMs();
1245
+ int waitMs = (int)Math.Max(150, Math.Min(1200, budget - 150));
1246
+ ProcessStartInfo psi = new ProcessStartInfo(exe, "--hwnd " + hwnd.ToInt64().ToString() + " --out \"" + tmp + "\" --timeout " + waitMs);
1247
+ psi.UseShellExecute = false;
1248
+ psi.CreateNoWindow = true;
1249
+ psi.RedirectStandardOutput = true;
1250
+ psi.RedirectStandardError = true;
1251
+ using (Process p = Process.Start(psi))
1252
+ {
1253
+ if (p == null) return null;
1254
+ if (!p.WaitForExit(waitMs + 900)) { try { p.Kill(); } catch { } return null; }
1255
+ if (p.ExitCode != 0 || !File.Exists(tmp)) return null;
1256
+ byte[] bytes = File.ReadAllBytes(tmp);
1257
+ using (MemoryStream ms = new MemoryStream(bytes)) return new Bitmap(ms);
1258
+ }
1259
+ }
1260
+ finally { try { if (File.Exists(tmp)) File.Delete(tmp); } catch { } }
1261
+ }
1262
+ catch { return null; }
1263
+ }
1264
+
1265
+ Dictionary<string, object> CaptureWindow(IntPtr h)
1266
+ {
1267
+ RECT r = BoundsOf(h);
1268
+ int w = Math.Max(1, r.Right - r.Left);
1269
+ int ht = Math.Max(1, r.Bottom - r.Top);
1270
+ // Cap pathological sizes so a maximised multi-monitor window cannot blow the
1271
+ // attachment byte budget.
1272
+ const int cap = 2560;
1273
+ double scale = 1.0;
1274
+ if (w > cap || ht > cap)
1275
+ {
1276
+ scale = Math.Min((double)cap / w, (double)cap / ht);
1277
+ w = Math.Max(1, (int)(w * scale));
1278
+ ht = Math.Max(1, (int)(ht * scale));
1279
+ }
1280
+
1281
+ Bitmap wgc = TryCaptureWithWgc(h);
1282
+ if (wgc != null)
1283
+ {
1284
+ Dictionary<string, object> wg = FinishShot(h, wgc, r, true, 1.0, 1.0);
1285
+ return wg;
1286
+ }
1287
+
1288
+ Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
1289
+ bool printed = false;
1290
+ using (Graphics g = Graphics.FromImage(bmp))
1291
+ {
1292
+ IntPtr hdc = g.GetHdc();
1293
+ try { printed = PrintWindow(h, hdc, 2); }
1294
+ finally { g.ReleaseHdc(hdc); }
1295
+ }
1296
+ if (!printed || IsBlank(bmp))
1297
+ {
1298
+ bmp.Dispose();
1299
+ bmp = CaptureFromScreen(r, w, ht);
1300
+ }
1301
+
1302
+ // DPI-virtualized targets render at their own (usually 96 DPI) size while
1303
+ // occupying a larger physical rect, so PrintWindow leaves a black halo.
1304
+ // Crop to the painted region and remember the physical/screenshot ratio so
1305
+ // coordinate actions can map screenshot pixels back to screen pixels.
1306
+ int fullWidth = bmp.Width, fullHeight = bmp.Height;
1307
+ Rectangle painted = PaintedBounds(bmp);
1308
+ if (painted.Width > 16 && painted.Height > 16 && (painted.Width < bmp.Width || painted.Height < bmp.Height))
1309
+ {
1310
+ Bitmap cropped = bmp.Clone(new Rectangle(0, 0, painted.Right, painted.Bottom), bmp.PixelFormat);
1311
+ bmp.Dispose();
1312
+ bmp = cropped;
1313
+ w = bmp.Width;
1314
+ ht = bmp.Height;
1315
+ }
1316
+ double scaleX = (double)fullWidth / Math.Max(1, bmp.Width);
1317
+ double scaleY = (double)fullHeight / Math.Max(1, bmp.Height);
1318
+
1319
+ byte[] png;
1320
+ using (MemoryStream ms = new MemoryStream())
1321
+ {
1322
+ bmp.Save(ms, ImageFormat.Png);
1323
+ png = ms.ToArray();
1324
+ }
1325
+ bmp.Dispose();
1326
+
1327
+ Dictionary<string, object> d = new Dictionary<string, object>();
1328
+ d["mediaType"] = "image/png";
1329
+ d["base64"] = Convert.ToBase64String(png);
1330
+ d["width"] = (long)w;
1331
+ d["height"] = (long)ht;
1332
+ d["originX"] = (long)r.Left;
1333
+ d["originY"] = (long)r.Top;
1334
+ d["zIndex"] = 1L;
1335
+ d["printed"] = printed;
1336
+ d["scaleX"] = scaleX;
1337
+ d["scaleY"] = scaleY;
1338
+ return d;
1339
+ }
1340
+
1341
+ static Dictionary<string, object> FinishShot(IntPtr h, Bitmap bmp, RECT r, bool printed, double scaleX, double scaleY)
1342
+ {
1343
+ int w = bmp.Width, ht = bmp.Height;
1344
+ byte[] png;
1345
+ using (MemoryStream ms = new MemoryStream())
1346
+ {
1347
+ bmp.Save(ms, ImageFormat.Png);
1348
+ png = ms.ToArray();
1349
+ }
1350
+ bmp.Dispose();
1351
+ Dictionary<string, object> d = new Dictionary<string, object>();
1352
+ d["mediaType"] = "image/png";
1353
+ d["base64"] = Convert.ToBase64String(png);
1354
+ d["width"] = (long)w;
1355
+ d["height"] = (long)ht;
1356
+ d["originX"] = (long)r.Left;
1357
+ d["originY"] = (long)r.Top;
1358
+ d["zIndex"] = 1L;
1359
+ d["printed"] = printed;
1360
+ d["scaleX"] = scaleX;
1361
+ d["scaleY"] = scaleY;
1362
+ return d;
1363
+ }
1364
+
1365
+ static Bitmap CaptureFromScreen(RECT r, int w, int ht)
1366
+ {
1367
+ Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
1368
+ using (Graphics g = Graphics.FromImage(bmp))
1369
+ {
1370
+ try { g.CopyFromScreen(r.Left, r.Top, 0, 0, new Size(w, ht), CopyPixelOperation.SourceCopy); }
1371
+ catch { }
1372
+ }
1373
+ return bmp;
1374
+ }
1375
+
1376
+ /// Bounding box of everything that is not uniform black, i.e. the region the
1377
+ /// target actually painted. Used to strip the DPI-virtualization halo.
1378
+ static Rectangle PaintedBounds(Bitmap bmp)
1379
+ {
1380
+ int right = 0, bottom = 0;
1381
+ int step = Math.Max(1, Math.Min(bmp.Width, bmp.Height) / 200);
1382
+ for (int y = 0; y < bmp.Height; y += step)
1383
+ {
1384
+ for (int x = 0; x < bmp.Width; x += step)
1385
+ {
1386
+ Color c = bmp.GetPixel(x, y);
1387
+ if (c.R > 8 || c.G > 8 || c.B > 8)
1388
+ {
1389
+ if (x > right) right = x;
1390
+ if (y > bottom) bottom = y;
1391
+ }
1392
+ }
1393
+ }
1394
+ return new Rectangle(0, 0, Math.Min(bmp.Width, right + step), Math.Min(bmp.Height, bottom + step));
1395
+ }
1396
+
1397
+ static bool IsBlank(Bitmap bmp)
1398
+ {
1399
+ int samples = 0, dark = 0;
1400
+ int stepX = Math.Max(1, bmp.Width / 12);
1401
+ int stepY = Math.Max(1, bmp.Height / 12);
1402
+ for (int y = 0; y < bmp.Height; y += stepY)
1403
+ {
1404
+ for (int x = 0; x < bmp.Width; x += stepX)
1405
+ {
1406
+ Color c = bmp.GetPixel(x, y);
1407
+ samples++;
1408
+ if (c.R < 6 && c.G < 6 && c.B < 6) dark++;
1409
+ }
1410
+ }
1411
+ return samples > 0 && dark == samples;
1412
+ }
1413
+
1414
+ // ---- element resolution ----
1415
+ AutomationElement ElementAt(IntPtr hwnd, long index)
1416
+ {
1417
+ Snapshot snap;
1418
+ lock (_sync)
1419
+ {
1420
+ if (!_snapshots.TryGetValue(hwnd.ToInt64(), out snap))
1421
+ throw new ToolError("stale_snapshot", "no observation for this window yet: call get_window_state with include_text=true before using element_index");
1422
+ }
1423
+ if ((DateTime.UtcNow - snap.At).TotalMilliseconds > _snapshotTtlMs)
1424
+ throw new ToolError("stale_snapshot", "the element snapshot is older than " + _snapshotTtlMs + "ms: call get_window_state again before acting on element indexes");
1425
+ AutomationElement el;
1426
+ if (!snap.Elements.TryGetValue(index, out el))
1427
+ throw new ToolError("bad_index", "element index " + index + " is not part of the latest observation for this window");
1428
+ return el;
1429
+ }
1430
+
1431
+ static Dictionary<string, object> ScreenshotTarget(IntPtr h, AutomationElement el)
1432
+ {
1433
+ RECT obs = BoundsOf(h);
1434
+ System.Windows.Rect r = el.Current.BoundingRectangle;
1435
+ if (r.IsEmpty || r.Width <= 0 || r.Height <= 0)
1436
+ throw new ToolError("no_geometry", "the element has no bounding rectangle; use x/y coordinates instead");
1437
+ Dictionary<string, object> d = new Dictionary<string, object>();
1438
+ d["screenX"] = (int)r.X + (int)(r.Width / 2);
1439
+ d["screenY"] = (int)r.Y + (int)(r.Height / 2);
1440
+ d["originX"] = obs.Left;
1441
+ d["originY"] = obs.Top;
1442
+ return d;
1443
+ }
1444
+
1445
+ // ---- input ----
1446
+ static void SendMouse(uint flags) { SendMouseData(flags, 0); }
1447
+
1448
+ static void SendMouseData(uint flags, int data)
1449
+ {
1450
+ INPUT inp = new INPUT();
1451
+ inp.type = INPUT_MOUSE;
1452
+ inp.u.mi.dx = 0; inp.u.mi.dy = 0;
1453
+ inp.u.mi.mouseData = (uint)data;
1454
+ inp.u.mi.dwFlags = flags;
1455
+ INPUT[] arr = new INPUT[] { inp };
1456
+ SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1457
+ }
1458
+
1459
+ static void KeyDown(ushort vk) { SendKey(vk, 0, 0); }
1460
+ static void KeyUp(ushort vk) { SendKey(vk, 0, KEYEVENTF_KEYUP); }
1461
+
1462
+ static void SendKey(ushort vk, ushort scan, uint flags)
1463
+ {
1464
+ INPUT inp = new INPUT();
1465
+ inp.type = INPUT_KEYBOARD;
1466
+ inp.u.ki.wVk = vk;
1467
+ inp.u.ki.wScan = scan;
1468
+ inp.u.ki.dwFlags = flags;
1469
+ INPUT[] arr = new INPUT[] { inp };
1470
+ SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1471
+ }
1472
+
1473
+ static void SendUnicode(char ch, bool up)
1474
+ {
1475
+ INPUT inp = new INPUT();
1476
+ inp.type = INPUT_KEYBOARD;
1477
+ inp.u.ki.wVk = 0;
1478
+ inp.u.ki.wScan = ch;
1479
+ inp.u.ki.dwFlags = KEYEVENTF_UNICODE | (up ? KEYEVENTF_KEYUP : 0);
1480
+ INPUT[] arr = new INPUT[] { inp };
1481
+ SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
1482
+ }
1483
+
1484
+ void ClickAt(int screenX, int screenY, int count, string button, Dictionary<string, object> p)
1485
+ {
1486
+ SetCursorPos(screenX, screenY);
1487
+ System.Threading.Thread.Sleep(35);
1488
+ for (int i = 0; i < Math.Max(1, count); i++)
1489
+ {
1490
+ CheckCancel(p);
1491
+ if (i > 0) System.Threading.Thread.Sleep(60);
1492
+ uint down = MOUSEEVENTF_LEFTDOWN, up = MOUSEEVENTF_LEFTUP;
1493
+ if (button == "r") { down = MOUSEEVENTF_RIGHTDOWN; up = MOUSEEVENTF_RIGHTUP; }
1494
+ else if (button == "m") { down = MOUSEEVENTF_MIDDLEDOWN; up = MOUSEEVENTF_MIDDLEUP; }
1495
+ SendMouse(down);
1496
+ try { System.Threading.Thread.Sleep(22); }
1497
+ finally { SendMouse(up); }
1498
+ }
1499
+ }
1500
+
1501
+ public Dictionary<string, object> Click(Dictionary<string, object> p)
1502
+ {
1503
+ CheckCancel(p);
1504
+ AssertUnlocked();
1505
+ EnsureOverlay(8000);
1506
+ IntPtr h = ResolveWindow(p);
1507
+ AssertAppApproved(h);
1508
+ long elementIndex = Json.GetLong(p, "element_index", -1);
1509
+ string button = NormalizeButton(Json.GetString(p, "mouse_button"));
1510
+ int count = (int)Json.GetLong(p, "click_count", 1);
1511
+ int sx, sy;
1512
+
1513
+ if (elementIndex >= 0)
1514
+ {
1515
+ AutomationElement el = ElementAt(h, elementIndex);
1516
+ Dictionary<string, object> t = ScreenshotTarget(h, el);
1517
+ sx = (int)t["screenX"]; sy = (int)t["screenY"];
1518
+ // Prefer the semantic invoke when the element exposes one and we are
1519
+ // doing a plain left single click: it survives occlusion and does not
1520
+ // depend on pixel geometry.
1521
+ if (count == 1 && button == "l")
1522
+ {
1523
+ object pattern;
1524
+ try
1525
+ {
1526
+ if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1527
+ {
1528
+ ((InvokePattern)pattern).Invoke();
1529
+ Dictionary<string, object> r2 = new Dictionary<string, object>();
1530
+ r2["clicked"] = "invoke"; r2["element_index"] = elementIndex;
1531
+ return r2;
1532
+ }
1533
+ }
1534
+ catch (Exception) { }
1535
+ }
1536
+ }
1537
+ else
1538
+ {
1539
+ double x = Json.GetDouble(p, "x", double.NaN);
1540
+ double y = Json.GetDouble(p, "y", double.NaN);
1541
+ if (double.IsNaN(x) || double.IsNaN(y)) throw new ToolError("bad_request", "click needs either 'element_index' or both 'x' and 'y' (window-relative pixels)");
1542
+ string screenshotId = Json.GetString(p, "screenshotId");
1543
+ int originX, originY;
1544
+ double coordScaleX = 1.0, coordScaleY = 1.0;
1545
+ if (!string.IsNullOrEmpty(screenshotId))
1546
+ {
1547
+ ShotRegion region = ResolveShot(h, screenshotId);
1548
+ AssertPointInsideShot(x, y, region, screenshotId, "click");
1549
+ originX = region.OriginX; originY = region.OriginY;
1550
+ coordScaleX = region.ScaleX; coordScaleY = region.ScaleY;
1551
+ }
1552
+ else
1553
+ {
1554
+ RECT bounds = BoundsOf(h);
1555
+ originX = bounds.Left; originY = bounds.Top;
1556
+ }
1557
+ sx = originX + (int)Math.Round(x * coordScaleX);
1558
+ sy = originY + (int)Math.Round(y * coordScaleY);
1559
+ }
1560
+
1561
+ Activate(h); // pointer clicks need a real foreground window
1562
+ MarkCursor(sx, sy);
1563
+ CheckCancel(p);
1564
+ ClickAt(sx, sy, count, button, p);
1565
+ Dictionary<string, object> r = new Dictionary<string, object>();
1566
+ r["clicked"] = "pointer";
1567
+ r["screenX"] = (long)sx;
1568
+ r["screenY"] = (long)sy;
1569
+ r["clickCount"] = (long)count;
1570
+ return r;
1571
+ }
1572
+
1573
+ static string NormalizeButton(string b)
1574
+ {
1575
+ if (string.IsNullOrEmpty(b)) return "l";
1576
+ string s = b.Trim().ToLowerInvariant();
1577
+ if (s == "right" || s == "r") return "r";
1578
+ if (s == "middle" || s == "m") return "m";
1579
+ return "l";
1580
+ }
1581
+
1582
+ public Dictionary<string, object> PressKey(Dictionary<string, object> p)
1583
+ {
1584
+ CheckCancel(p);
1585
+ AssertUnlocked();
1586
+ EnsureOverlay(8000);
1587
+ IntPtr h = ResolveWindow(p);
1588
+ AssertAppApproved(h);
1589
+ string key = Json.GetString(p, "key");
1590
+ if (string.IsNullOrEmpty(key)) throw new ToolError("bad_request", "press_key requires 'key'");
1591
+ Activate(h);
1592
+ Chord(key, p);
1593
+ Dictionary<string, object> r = new Dictionary<string, object>();
1594
+ r["pressed"] = key;
1595
+ return r;
1596
+ }
1597
+
1598
+ static readonly Dictionary<string, ushort> KeyMap = BuildKeyMap();
1599
+
1600
+ static Dictionary<string, ushort> BuildKeyMap()
1601
+ {
1602
+ Dictionary<string, ushort> m = new Dictionary<string, ushort>(StringComparer.OrdinalIgnoreCase);
1603
+ for (char c = 'a'; c <= 'z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'a'));
1604
+ for (char c = 'A'; c <= 'Z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'A'));
1605
+ for (char c = '0'; c <= '9'; c++) m[c.ToString()] = (ushort)(0x30 + (c - '0'));
1606
+ m["return"] = 0x0D; m["enter"] = 0x0D; m["kp_enter"] = 0x0D;
1607
+ m["tab"] = 0x09; m["space"] = 0x20; m["backspace"] = 0x08; m["escape"] = 0x1B; m["esc"] = 0x1B;
1608
+ m["delete"] = 0x2E; m["insert"] = 0x2D; m["home"] = 0x24; m["end"] = 0x23;
1609
+ m["page_up"] = 0x21; m["pageup"] = 0x21; m["prior"] = 0x21; m["page_down"] = 0x22; m["pagedown"] = 0x22; m["next"] = 0x22;
1610
+ m["left"] = 0x25; m["up"] = 0x26; m["right"] = 0x27; m["down"] = 0x28;
1611
+ m["shift_l"] = 0xA0; m["shift"] = 0xA0; m["shift_r"] = 0xA1;
1612
+ m["control_l"] = 0xA2; m["control"] = 0xA2; m["ctrl"] = 0xA2; m["control_r"] = 0xA3;
1613
+ m["alt_l"] = 0xA4; m["alt"] = 0xA4; m["alt_r"] = 0xA5;
1614
+ m["super_l"] = 0x5B; m["win"] = 0x5B; m["meta"] = 0x5B; m["lwin"] = 0x5B; m["super"] = 0x5B;
1615
+ m["super_r"] = 0x5C; m["rwin"] = 0x5C;
1616
+ m["caps_lock"] = 0x14; m["num_lock"] = 0x90; m["scroll_lock"] = 0x91;
1617
+ m["menu"] = 0x5D; m["apps"] = 0x5D; m["print"] = 0x2C; m["pause"] = 0x13;
1618
+ m["period"] = 0xBE; m["greater"] = 0xBE; m["comma"] = 0xBC; m["less"] = 0xBC;
1619
+ m["slash"] = 0xBF; m["question"] = 0xBF; m["backslash"] = 0xDC; m["bar"] = 0xDC;
1620
+ m["semicolon"] = 0xBA; m["colon"] = 0xBA; m["apostrophe"] = 0xDE; m["quote"] = 0xDE;
1621
+ m["minus"] = 0xBD; m["underscore"] = 0xBD; m["equal"] = 0xBB; m["plus"] = 0xBB;
1622
+ m["bracketleft"] = 0xDB; m["bracketright"] = 0xDD; m["grave"] = 0xC0; m["asciitilde"] = 0xC0;
1623
+ for (int i = 1; i <= 24; i++) { m["f" + i] = (ushort)(0x6F + i); }
1624
+ // Numpad: both X keysym style (KP_0) and the aliases the API documents.
1625
+ for (int i = 0; i <= 9; i++)
1626
+ {
1627
+ m["kp_" + i] = (ushort)(0x60 + i);
1628
+ m["numpad_" + i] = (ushort)(0x60 + i);
1629
+ }
1630
+ m["kp_add"] = 0x6B; m["numpad_add"] = 0x6B; m["plus_numpad"] = 0x6B;
1631
+ m["kp_subtract"] = 0x6D; m["numpad_subtract"] = 0x6D; m["minus_numpad"] = 0x6D;
1632
+ m["kp_multiply"] = 0x6A; m["numpad_multiply"] = 0x6A; m["kp_asterisk"] = 0x6A;
1633
+ m["kp_divide"] = 0x6F; m["numpad_divide"] = 0x6F; m["kp_slash"] = 0x6F;
1634
+ m["kp_decimal"] = 0x6E; m["numpad_decimal"] = 0x6E; m["kp_separator"] = 0x6C;
1635
+ m["numpad_enter"] = 0x0D;
1636
+ return m;
1637
+ }
1638
+
1639
+ static readonly HashSet<ushort> ExtendedKeys = new HashSet<ushort>(new ushort[] {
1640
+ 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x2D,0x2E,0x5B,0x5C,0x5D,0x90,0x6F,0xA3,0xA5
1641
+ });
1642
+
1643
+ static void SendVk(ushort vk, bool up)
1644
+ {
1645
+ uint flags = up ? KEYEVENTF_KEYUP : 0;
1646
+ if (ExtendedKeys.Contains(vk)) flags |= KEYEVENTF_EXTENDEDKEY;
1647
+ SendKey(vk, 0, flags);
1648
+ }
1649
+
1650
+ void Chord(string chord, Dictionary<string, object> p)
1651
+ {
1652
+ string[] parts = chord.Split('+');
1653
+ List<ushort> keys = new List<ushort>();
1654
+ foreach (string raw in parts)
1655
+ {
1656
+ string token = raw.Trim();
1657
+ if (token.Length == 0) continue;
1658
+ keys.Add(ResolveKey(token));
1659
+ }
1660
+ if (keys.Count == 0) throw new ToolError("bad_request", "empty key chord");
1661
+ int pressed = 0;
1662
+ try
1663
+ {
1664
+ for (int i = 0; i < keys.Count; i++)
1665
+ {
1666
+ CheckCancel(p);
1667
+ SendVk(keys[i], false); pressed++;
1668
+ System.Threading.Thread.Sleep(12);
1669
+ }
1670
+ }
1671
+ finally { for (int i = pressed - 1; i >= 0; i--) SendVk(keys[i], true); }
1672
+ }
1673
+
1674
+ /// The Windows/Meta key family is hard-denied: it opens Run, Settings and
1675
+ /// shell surfaces that Computer Use must never drive. Enforced in the helper
1676
+ /// as well as in the plugin, so a direct protocol call cannot bypass it.
1677
+ static readonly HashSet<string> DeniedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
1678
+ "win", "win_l", "win_r", "lwin", "rwin", "windows", "windows_l", "windows_r",
1679
+ "super", "super_l", "super_r", "meta", "meta_l", "meta_r", "cmd", "command", "os", "os_l", "os_r"
1680
+ };
1681
+
1682
+ static ushort ResolveKey(string token)
1683
+ {
1684
+ if (DeniedKeys.Contains(token))
1685
+ throw new ToolError("hard_denied", "the Windows/Meta/Super key ('" + token + "') is hard-denied: it opens Run, Settings and shell surfaces that Computer Use must never drive.");
1686
+ ushort vk;
1687
+ if (KeyMap.TryGetValue(token, out vk)) return vk;
1688
+ if (token.Length == 1)
1689
+ {
1690
+ char c = token[0];
1691
+ short scan = VkKeyScan(c);
1692
+ if (scan != -1) return (ushort)(scan & 0xFF);
1693
+ return (ushort)char.ToUpperInvariant(c);
1694
+ }
1695
+ throw new ToolError("bad_key", "unknown key name '" + token + "'");
1696
+ }
1697
+
1698
+ [DllImport("user32.dll")] static extern short VkKeyScan(char ch);
1699
+
1700
+ public Dictionary<string, object> TypeText(Dictionary<string, object> p)
1701
+ {
1702
+ CheckCancel(p);
1703
+ AssertUnlocked();
1704
+ EnsureOverlay(8000);
1705
+ IntPtr h = ResolveWindow(p);
1706
+ AssertAppApproved(h);
1707
+ string text = Json.GetString(p, "text");
1708
+ if (text == null) throw new ToolError("bad_request", "type_text requires 'text'");
1709
+ Activate(h);
1710
+ for (int i = 0; i < text.Length; i++)
1711
+ {
1712
+ CheckCancel(p);
1713
+ char c = text[i];
1714
+ if (c == '\n') { SendVk(0x0D, false); SendVk(0x0D, true); continue; }
1715
+ if (c == '\t') { SendVk(0x09, false); SendVk(0x09, true); continue; }
1716
+ if (c == '\r') continue;
1717
+ SendUnicode(c, false);
1718
+ SendUnicode(c, true);
1719
+ if ((i & 15) == 0) System.Threading.Thread.Sleep(4);
1720
+ }
1721
+ Dictionary<string, object> r = new Dictionary<string, object>();
1722
+ r["typed"] = (long)text.Length;
1723
+ return r;
1724
+ }
1725
+
1726
+ public Dictionary<string, object> Scroll(Dictionary<string, object> p)
1727
+ {
1728
+ CheckCancel(p);
1729
+ AssertUnlocked();
1730
+ EnsureOverlay(8000);
1731
+ IntPtr h = ResolveWindow(p);
1732
+ AssertAppApproved(h);
1733
+ // Validation first: an invalid call must not activate a window or move the pointer.
1734
+ double scrollY = Json.GetDouble(p, "scrollY", 0);
1735
+ double scrollX = Json.GetDouble(p, "scrollX", 0);
1736
+ double x = Json.GetDouble(p, "x", double.NaN);
1737
+ double y = Json.GetDouble(p, "y", double.NaN);
1738
+ if (double.IsNaN(x) || double.IsNaN(y))
1739
+ throw new ToolError("bad_request", "scroll requires finite 'x' and 'y' (the point to scroll from, in window/screenshot coordinates); click a pane first if it needs focus");
1740
+ if (Math.Abs(scrollX) < 0.0001 && Math.Abs(scrollY) < 0.0001)
1741
+ throw new ToolError("bad_request", "scroll requires a non-zero scrollX or scrollY");
1742
+ Activate(h);
1743
+ int sx, sy;
1744
+ string scrollShot = Json.GetString(p, "screenshotId");
1745
+ int originLeft, originTop;
1746
+ double scrollScaleX = 1.0, scrollScaleY = 1.0;
1747
+ if (!string.IsNullOrEmpty(scrollShot))
1748
+ {
1749
+ ShotRegion region = ResolveShot(h, scrollShot);
1750
+ AssertPointInsideShot(x, y, region, scrollShot, "scroll origin");
1751
+ originLeft = region.OriginX; originTop = region.OriginY;
1752
+ scrollScaleX = region.ScaleX; scrollScaleY = region.ScaleY;
1753
+ }
1754
+ else
1755
+ {
1756
+ RECT b = BoundsOf(h);
1757
+ originLeft = b.Left; originTop = b.Top;
1758
+ }
1759
+ sx = originLeft + (int)Math.Round(x * scrollScaleX);
1760
+ sy = originTop + (int)Math.Round(y * scrollScaleY);
1761
+ MarkCursor(sx, sy);
1762
+ SetCursorPos(sx, sy);
1763
+ System.Threading.Thread.Sleep(40);
1764
+ if (Math.Abs(scrollY) > 0.0001) SendMouseData(MOUSEEVENTF_WHEEL, (int)(-scrollY * 120));
1765
+ if (Math.Abs(scrollX) > 0.0001) SendMouseData(MOUSEEVENTF_HWHEEL, (int)(-scrollX * 120));
1766
+ Dictionary<string, object> r = new Dictionary<string, object>();
1767
+ r["scrolled"] = true;
1768
+ r["screenX"] = (long)sx; r["screenY"] = (long)sy;
1769
+ return r;
1770
+ }
1771
+
1772
+ public Dictionary<string, object> Drag(Dictionary<string, object> p)
1773
+ {
1774
+ CheckCancel(p);
1775
+ AssertUnlocked();
1776
+ EnsureOverlay(8000);
1777
+ IntPtr h = ResolveWindow(p);
1778
+ AssertAppApproved(h);
1779
+ double fromX = Json.GetDouble(p, "from_x", double.NaN);
1780
+ double fromY = Json.GetDouble(p, "from_y", double.NaN);
1781
+ double toX = Json.GetDouble(p, "to_x", double.NaN);
1782
+ double toY = Json.GetDouble(p, "to_y", double.NaN);
1783
+ if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
1784
+ throw new ToolError("bad_request", "drag requires finite from_x/from_y/to_x/to_y (window-relative pixels)");
1785
+ Activate(h);
1786
+ int originX, originY;
1787
+ double dragScaleX = 1.0, dragScaleY = 1.0;
1788
+ string dragShot = Json.GetString(p, "screenshotId");
1789
+ if (!string.IsNullOrEmpty(dragShot))
1790
+ {
1791
+ ShotRegion region = ResolveShot(h, dragShot);
1792
+ AssertPointInsideShot(fromX, fromY, region, dragShot, "drag start");
1793
+ AssertPointInsideShot(toX, toY, region, dragShot, "drag end");
1794
+ originX = region.OriginX; originY = region.OriginY;
1795
+ dragScaleX = region.ScaleX; dragScaleY = region.ScaleY;
1796
+ }
1797
+ else
1798
+ {
1799
+ RECT b = BoundsOf(h);
1800
+ originX = b.Left; originY = b.Top;
1801
+ }
1802
+ int fx = originX + (int)Math.Round(fromX * dragScaleX);
1803
+ int fy = originY + (int)Math.Round(fromY * dragScaleY);
1804
+ int tx = originX + (int)Math.Round(toX * dragScaleX);
1805
+ int ty = originY + (int)Math.Round(toY * dragScaleY);
1806
+
1807
+ MarkCursor(fx, fy);
1808
+ SetCursorPos(fx, fy);
1809
+ System.Threading.Thread.Sleep(60);
1810
+ CheckCancel(p);
1811
+ SendMouse(MOUSEEVENTF_LEFTDOWN);
1812
+ try
1813
+ {
1814
+ System.Threading.Thread.Sleep(80);
1815
+ const int steps = 14;
1816
+ for (int i = 1; i <= steps; i++)
1817
+ {
1818
+ CheckCancel(p);
1819
+ int x = fx + (tx - fx) * i / steps;
1820
+ int y = fy + (ty - fy) * i / steps;
1821
+ SetCursorPos(x, y);
1822
+ System.Threading.Thread.Sleep(18);
1823
+ }
1824
+ System.Threading.Thread.Sleep(90);
1825
+ CheckCancel(p);
1826
+ }
1827
+ finally { SendMouse(MOUSEEVENTF_LEFTUP); }
1828
+
1829
+ Dictionary<string, object> r = new Dictionary<string, object>();
1830
+ r["dragged"] = true;
1831
+ r["from"] = new long[] { fx, fy };
1832
+ r["to"] = new long[] { tx, ty };
1833
+ return r;
1834
+ }
1835
+
1836
+ public Dictionary<string, object> SetValue(Dictionary<string, object> p)
1837
+ {
1838
+ CheckCancel(p);
1839
+ AssertUnlocked();
1840
+ EnsureOverlay(8000);
1841
+ IntPtr h = ResolveWindow(p);
1842
+ AssertAppApproved(h);
1843
+ long elementIndex = Json.GetLong(p, "element_index", -1);
1844
+ if (elementIndex < 0) throw new ToolError("bad_request", "set_value requires 'element_index'");
1845
+ string value = Json.GetString(p, "value");
1846
+ if (value == null) throw new ToolError("bad_request", "set_value requires 'value'");
1847
+ AutomationElement el = ElementAt(h, elementIndex);
1848
+ object pattern;
1849
+ if (!el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern))
1850
+ throw new ToolError("pattern_unavailable", "element [" + elementIndex + "] does not expose the Value pattern; use click + type_text instead");
1851
+ ((ValuePattern)pattern).SetValue(value);
1852
+ Dictionary<string, object> r = new Dictionary<string, object>();
1853
+ r["valueSet"] = true;
1854
+ r["element_index"] = elementIndex;
1855
+ return r;
1856
+ }
1857
+
1858
+ public Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p)
1859
+ {
1860
+ CheckCancel(p);
1861
+ AssertUnlocked();
1862
+ EnsureOverlay(8000);
1863
+ IntPtr h = ResolveWindow(p);
1864
+ AssertAppApproved(h);
1865
+ long elementIndex = Json.GetLong(p, "element_index", -1);
1866
+ string action = Json.GetString(p, "action");
1867
+ if (elementIndex < 0 || string.IsNullOrEmpty(action))
1868
+ throw new ToolError("bad_request", "perform_secondary_action requires 'element_index' and 'action'");
1869
+ AutomationElement el = ElementAt(h, elementIndex);
1870
+ string a = action.Trim().ToLowerInvariant();
1871
+ object pattern;
1872
+
1873
+ if ((a == "raise" || a == "focus") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1874
+ { ((InvokePattern)pattern).Invoke(); }
1875
+ else if (a == "raise" || a == "focus") { try { el.SetFocus(); } catch (Exception ex) { throw new ToolError("action_failed", ex.Message); } }
1876
+ else if ((a == "invoke" || a == "press") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
1877
+ { ((InvokePattern)pattern).Invoke(); }
1878
+ else if ((a == "expand" || a == "collapse") && el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern))
1879
+ {
1880
+ ExpandCollapsePattern ec = (ExpandCollapsePattern)pattern;
1881
+ if (a == "expand") ec.Expand(); else ec.Collapse();
1882
+ }
1883
+ else if ((a == "toggle") && el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern))
1884
+ { ((TogglePattern)pattern).Toggle(); }
1885
+ else if ((a == "select" || a == "selectitem") && el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern))
1886
+ { ((SelectionItemPattern)pattern).Select(); }
1887
+ else if (a.StartsWith("scroll") && el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern))
1888
+ {
1889
+ ScrollPattern sp = (ScrollPattern)pattern;
1890
+ if (a.Contains("up")) sp.ScrollVertical(ScrollAmount.SmallDecrement);
1891
+ else if (a.Contains("down")) sp.ScrollVertical(ScrollAmount.SmallIncrement);
1892
+ else if (a.Contains("left")) sp.ScrollHorizontal(ScrollAmount.SmallDecrement);
1893
+ else if (a.Contains("right")) sp.ScrollHorizontal(ScrollAmount.SmallIncrement);
1894
+ else throw new ToolError("bad_action", "unsupported scroll direction in '" + action + "'");
1895
+ }
1896
+ else
1897
+ {
1898
+ throw new ToolError("action_unavailable", "element [" + elementIndex + "] does not expose action '" + action + "'");
1899
+ }
1900
+
1901
+ Dictionary<string, object> r = new Dictionary<string, object>();
1902
+ r["performed"] = action;
1903
+ r["element_index"] = elementIndex;
1904
+ return r;
1905
+ }
1906
+ }