deepseek-computer-use 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +16 -3
- package/lib/index.js +7 -0
- package/lib/update-client.js +408 -0
- package/native/Driver.cs +1906 -1906
- package/native/Json.cs +274 -274
- package/native/Overlay.cs +732 -732
- package/native/Program.cs +160 -160
- package/native/audio/Program.cs +95 -95
- package/native/wgcfw/Program.cs +249 -249
- package/package.json +20 -6
package/native/Driver.cs
CHANGED
|
@@ -1,1906 +1,1906 @@
|
|
|
1
|
-
|
|
2
|
-
using System;
|
|
3
|
-
using System.Collections.Generic;
|
|
4
|
-
using System.Diagnostics;
|
|
5
|
-
using System.Drawing;
|
|
6
|
-
using System.Drawing.Imaging;
|
|
7
|
-
using System.IO;
|
|
8
|
-
using System.Runtime.InteropServices;
|
|
9
|
-
using System.Text;
|
|
10
|
-
using System.Windows.Automation;
|
|
11
|
-
|
|
12
|
-
// ---------------------------------------------------------------------------
|
|
13
|
-
// Platform driver contract. Windows is the only implementation today; the
|
|
14
|
-
// interface exists so macOS (AX + CGEvent) and Linux (AT-SPI) drivers can be
|
|
15
|
-
// added without touching the DSH plugin layer or the stdio protocol.
|
|
16
|
-
// ---------------------------------------------------------------------------
|
|
17
|
-
public interface IPlatformDriver : IDisposable
|
|
18
|
-
{
|
|
19
|
-
string PlatformName { get; }
|
|
20
|
-
Dictionary<string, object> Ping(Dictionary<string, object> p);
|
|
21
|
-
Dictionary<string, object> Hello(Dictionary<string, object> p);
|
|
22
|
-
Dictionary<string, object> RestoreCursor(Dictionary<string, object> p);
|
|
23
|
-
void Maintenance();
|
|
24
|
-
Dictionary<string, object> Close(Dictionary<string, object> p);
|
|
25
|
-
Dictionary<string, object> ListWindows(Dictionary<string, object> p);
|
|
26
|
-
Dictionary<string, object> ListApps(Dictionary<string, object> p);
|
|
27
|
-
Dictionary<string, object> GetWindow(Dictionary<string, object> p);
|
|
28
|
-
Dictionary<string, object> LaunchApp(Dictionary<string, object> p);
|
|
29
|
-
Dictionary<string, object> GetWindowState(Dictionary<string, object> p);
|
|
30
|
-
Dictionary<string, object> Click(Dictionary<string, object> p);
|
|
31
|
-
Dictionary<string, object> PressKey(Dictionary<string, object> p);
|
|
32
|
-
Dictionary<string, object> TypeText(Dictionary<string, object> p);
|
|
33
|
-
Dictionary<string, object> Scroll(Dictionary<string, object> p);
|
|
34
|
-
Dictionary<string, object> Drag(Dictionary<string, object> p);
|
|
35
|
-
Dictionary<string, object> SetValue(Dictionary<string, object> p);
|
|
36
|
-
Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p);
|
|
37
|
-
Dictionary<string, object> ActivateWindow(Dictionary<string, object> p);
|
|
38
|
-
Dictionary<string, object> OverlayShow(Dictionary<string, object> p);
|
|
39
|
-
Dictionary<string, object> OverlayHide(Dictionary<string, object> p);
|
|
40
|
-
Dictionary<string, object> Configure(Dictionary<string, object> p);
|
|
41
|
-
Dictionary<string, object> ApproveApp(Dictionary<string, object> p);
|
|
42
|
-
Dictionary<string, object> EndTurn(Dictionary<string, object> p);
|
|
43
|
-
Dictionary<string, object> InvalidateState(Dictionary<string, object> p);
|
|
44
|
-
Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p);
|
|
45
|
-
Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/// Per-request time budget. Codex sends x-oai-cua-request-budget-ms with every call
|
|
49
|
-
/// and the helper honours it; here the budget bounds capture waits and is reported
|
|
50
|
-
/// back when a request overruns so the client and the helper agree on the deadline.
|
|
51
|
-
public static class Budget
|
|
52
|
-
{
|
|
53
|
-
static long _deadlineTicks;
|
|
54
|
-
static string _cancelFile;
|
|
55
|
-
|
|
56
|
-
public static void Begin(long ms, string cancelFile)
|
|
57
|
-
{
|
|
58
|
-
_deadlineTicks = ms > 0 ? DateTime.UtcNow.Ticks + ms * TimeSpan.TicksPerMillisecond : 0;
|
|
59
|
-
_cancelFile = cancelFile;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
public static void Check()
|
|
63
|
-
{
|
|
64
|
-
if (_cancelFile != null && File.Exists(_cancelFile))
|
|
65
|
-
{
|
|
66
|
-
string reason = "cancelled";
|
|
67
|
-
try { reason = File.ReadAllText(_cancelFile).Trim(); } catch { }
|
|
68
|
-
if (reason != "turn_ended" && reason != "timeout") reason = "cancelled";
|
|
69
|
-
throw new ToolError(reason, "the native request was stopped; take a fresh observation before continuing");
|
|
70
|
-
}
|
|
71
|
-
if (Expired()) throw new ToolError("timeout", "the native request exceeded its time budget");
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
public static long RemainingMs()
|
|
75
|
-
{
|
|
76
|
-
if (_deadlineTicks == 0) return long.MaxValue / 4;
|
|
77
|
-
return (_deadlineTicks - DateTime.UtcNow.Ticks) / TimeSpan.TicksPerMillisecond;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
public static bool Expired()
|
|
81
|
-
{
|
|
82
|
-
return _deadlineTicks != 0 && RemainingMs() <= 0;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
public class ToolError : Exception
|
|
87
|
-
{
|
|
88
|
-
public readonly string Code;
|
|
89
|
-
public ToolError(string code, string message) : base(message) { Code = code; }
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// ---------------------------------------------------------------------------
|
|
93
|
-
// Windows driver: Win32 + UI Automation + PrintWindow.
|
|
94
|
-
// ---------------------------------------------------------------------------
|
|
95
|
-
public class WindowsDriver : IPlatformDriver
|
|
96
|
-
{
|
|
97
|
-
public string PlatformName { get { return "windows"; } }
|
|
98
|
-
|
|
99
|
-
// ---- win32 ----
|
|
100
|
-
delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
|
|
101
|
-
|
|
102
|
-
[DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc cb, IntPtr lParam);
|
|
103
|
-
[DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
|
|
104
|
-
[DllImport("user32.dll")] static extern bool IsWindow(IntPtr hWnd);
|
|
105
|
-
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowTextW(IntPtr hWnd, StringBuilder sb, int max);
|
|
106
|
-
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetClassNameW(IntPtr hWnd, StringBuilder sb, int max);
|
|
107
|
-
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
|
|
108
|
-
[DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT r);
|
|
109
|
-
[DllImport("user32.dll")] static extern bool GetClientRect(IntPtr hWnd, out RECT r);
|
|
110
|
-
[DllImport("user32.dll")] static extern bool PrintWindow(IntPtr hWnd, IntPtr hdc, uint flags);
|
|
111
|
-
[DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hWnd);
|
|
112
|
-
[DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hWnd, int cmd);
|
|
113
|
-
[DllImport("user32.dll")] static extern bool IsIconic(IntPtr hWnd);
|
|
114
|
-
[DllImport("user32.dll")] static extern bool SetCursorPos(int x, int y);
|
|
115
|
-
[DllImport("user32.dll")] static extern bool GetCursorPos(out POINT p);
|
|
116
|
-
[DllImport("user32.dll")] static extern uint SendInput(uint n, INPUT[] inputs, int size);
|
|
117
|
-
[DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
|
|
118
|
-
[DllImport("user32.dll")] static extern IntPtr SetFocus(IntPtr hWnd);
|
|
119
|
-
[DllImport("user32.dll")] static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach);
|
|
120
|
-
[DllImport("kernel32.dll")] static extern uint GetCurrentThreadId();
|
|
121
|
-
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid);
|
|
122
|
-
[DllImport("user32.dll")] static extern bool IsWindowEnabled(IntPtr hWnd);
|
|
123
|
-
[DllImport("user32.dll")] static extern bool SystemParametersInfo(uint action, uint param, IntPtr pv, uint winIni);
|
|
124
|
-
[DllImport("user32.dll")] static extern bool SwitchToThisWindow(IntPtr hWnd, bool fAltTab);
|
|
125
|
-
[DllImport("user32.dll")] static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
|
|
126
|
-
[DllImport("user32.dll")] static extern bool BringWindowToTop(IntPtr hWnd);
|
|
127
|
-
[DllImport("user32.dll")] static extern IntPtr SetThreadDpiAwarenessContext(IntPtr ctx);
|
|
128
|
-
[DllImport("user32.dll")] static extern int GetSystemMetrics(int index);
|
|
129
|
-
[DllImport("user32.dll")] static extern bool SetProcessDPIAware();
|
|
130
|
-
[DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT r, int size);
|
|
131
|
-
[DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr hWnd, uint cmd);
|
|
132
|
-
[DllImport("user32.dll")] static extern int GetWindowLong(IntPtr hWnd, int index);
|
|
133
|
-
|
|
134
|
-
[StructLayout(LayoutKind.Sequential)] struct RECT { public int Left, Top, Right, Bottom; }
|
|
135
|
-
[StructLayout(LayoutKind.Sequential)] struct POINT { public int X, Y; }
|
|
136
|
-
|
|
137
|
-
[StructLayout(LayoutKind.Sequential)]
|
|
138
|
-
struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
|
139
|
-
|
|
140
|
-
[StructLayout(LayoutKind.Sequential)]
|
|
141
|
-
struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
|
142
|
-
|
|
143
|
-
[StructLayout(LayoutKind.Explicit)]
|
|
144
|
-
struct INPUTUNION
|
|
145
|
-
{
|
|
146
|
-
[FieldOffset(0)] public MOUSEINPUT mi;
|
|
147
|
-
[FieldOffset(0)] public KEYBDINPUT ki;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
[StructLayout(LayoutKind.Sequential)]
|
|
151
|
-
struct INPUT { public uint type; public INPUTUNION u; }
|
|
152
|
-
|
|
153
|
-
const int INPUT_MOUSE = 0;
|
|
154
|
-
const int INPUT_KEYBOARD = 1;
|
|
155
|
-
const uint KEYEVENTF_KEYUP = 0x0002;
|
|
156
|
-
const uint KEYEVENTF_UNICODE = 0x0004;
|
|
157
|
-
const uint KEYEVENTF_EXTENDEDKEY = 0x0001;
|
|
158
|
-
const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
|
|
159
|
-
const uint MOUSEEVENTF_LEFTUP = 0x0004;
|
|
160
|
-
const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
|
|
161
|
-
const uint MOUSEEVENTF_RIGHTUP = 0x0010;
|
|
162
|
-
const uint MOUSEEVENTF_MIDDLEDOWN = 0x0020;
|
|
163
|
-
const uint MOUSEEVENTF_MIDDLEUP = 0x0040;
|
|
164
|
-
const uint MOUSEEVENTF_WHEEL = 0x0800;
|
|
165
|
-
const uint MOUSEEVENTF_HWHEEL = 0x1000;
|
|
166
|
-
const int DWMWA_EXTENDED_FRAME_BOUNDS = 9;
|
|
167
|
-
|
|
168
|
-
// ---- state ----
|
|
169
|
-
readonly object _sync = new object();
|
|
170
|
-
sealed class Snapshot
|
|
171
|
-
{
|
|
172
|
-
public long WindowId;
|
|
173
|
-
public DateTime At;
|
|
174
|
-
public Dictionary<long, AutomationElement> Elements = new Dictionary<long, AutomationElement>();
|
|
175
|
-
public Dictionary<long, string> FallbackLabels = new Dictionary<long, string>();
|
|
176
|
-
public List<object> Summary = new List<object>();
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
readonly Dictionary<long, Snapshot> _snapshots = new Dictionary<long, Snapshot>();
|
|
180
|
-
int _snapshotTtlMs = 120000;
|
|
181
|
-
|
|
182
|
-
/// A screenshot region the model was handed, so a later coordinate action can
|
|
183
|
-
/// be interpreted in that exact screenshot space (Codex: screenshotId).
|
|
184
|
-
sealed class ShotRegion
|
|
185
|
-
{
|
|
186
|
-
public long WindowId;
|
|
187
|
-
public int OriginX;
|
|
188
|
-
public int OriginY;
|
|
189
|
-
public int Width;
|
|
190
|
-
public int Height;
|
|
191
|
-
/// Physical pixels per screenshot pixel. 1.0 for DPI-aware targets; >1 when
|
|
192
|
-
/// the target is DPI-virtualized and PrintWindow returns an unscaled render.
|
|
193
|
-
public double ScaleX = 1.0;
|
|
194
|
-
public double ScaleY = 1.0;
|
|
195
|
-
public DateTime At;
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
// In-use indicator ("DSH 正在使用您的电脑" + whale-girl). Codex draws an animated
|
|
199
|
-
// border and a virtual cursor; this is the DSH equivalent, and Esc is the kill
|
|
200
|
-
// switch: the overlay drops a cancel file that every input path checks first.
|
|
201
|
-
Process _overlay;
|
|
202
|
-
System.Threading.Timer _cursorWatchdog;
|
|
203
|
-
DateTime _overlayUntil = DateTime.MinValue;
|
|
204
|
-
string _overlayCancelFile;
|
|
205
|
-
string _overlayText = "DSH 正在使用您的电脑";
|
|
206
|
-
string _overlayHint = "Esc 取消";
|
|
207
|
-
string _overlayAccent = "#3f7fd8";
|
|
208
|
-
string _cursorFile; // the overlay polls this to draw the virtual cursor
|
|
209
|
-
/// Turn token that the user interrupted with Esc. Codex persists the same
|
|
210
|
-
/// fact as a marker file per conversation+turn; here the token travels with
|
|
211
|
-
/// every input call, so the block ends exactly when the turn does.
|
|
212
|
-
string _interruptedTurn = null;
|
|
213
|
-
/// Apps the plugin approved through the DSH approval seam. Codex passes the same
|
|
214
|
-
/// fact per request as x-oai-cua-approved-app; here the helper is the enforcer.
|
|
215
|
-
readonly HashSet<string> _approvedApps = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
216
|
-
bool _requireAppApproval = false;
|
|
217
|
-
Process _audioProcess;
|
|
218
|
-
string _audioPath;
|
|
219
|
-
string _audioStopFile;
|
|
220
|
-
|
|
221
|
-
readonly Dictionary<string, ShotRegion> _shots = new Dictionary<string, ShotRegion>();
|
|
222
|
-
int _shotTtlMs = 120000;
|
|
223
|
-
int _shotSeq = 0;
|
|
224
|
-
readonly string _shotGeneration = Guid.NewGuid().ToString("N");
|
|
225
|
-
|
|
226
|
-
const uint GW_OWNER = 4;
|
|
227
|
-
const int GWL_STYLE = -16;
|
|
228
|
-
const int WS_POPUP = unchecked((int)0x80000000);
|
|
229
|
-
|
|
230
|
-
public WindowsDriver()
|
|
231
|
-
{
|
|
232
|
-
Maintenance(); // startup repair: a previous session may have died with cursors hidden
|
|
233
|
-
try { SetThreadDpiAwarenessContext(new IntPtr(-4)); } catch { }
|
|
234
|
-
try { SetProcessDPIAware(); } catch { }
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
public void Dispose()
|
|
238
|
-
{
|
|
239
|
-
if (_cursorWatchdog != null) { _cursorWatchdog.Dispose(); _cursorWatchdog = null; }
|
|
240
|
-
StopOverlay();
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
// ---- approval plumbing ----
|
|
244
|
-
public Dictionary<string, object> Configure(Dictionary<string, object> p)
|
|
245
|
-
{
|
|
246
|
-
_requireAppApproval = Json.GetBool(p, "requireAppApproval", _requireAppApproval);
|
|
247
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
248
|
-
r["requireAppApproval"] = _requireAppApproval;
|
|
249
|
-
return r;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
public Dictionary<string, object> ApproveApp(Dictionary<string, object> p)
|
|
253
|
-
{
|
|
254
|
-
string app = Json.GetString(p, "app");
|
|
255
|
-
if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "approve_app requires 'app'");
|
|
256
|
-
lock (_sync) { _approvedApps.Add(app); }
|
|
257
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
258
|
-
r["approved"] = app;
|
|
259
|
-
r["count"] = (long)_approvedApps.Count;
|
|
260
|
-
return r;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
/// Turn boundary: drop per-turn state (element snapshots, screenshot regions, the
|
|
264
|
-
/// Esc block, the indicator) but keep session-scoped app approvals.
|
|
265
|
-
public Dictionary<string, object> EndTurn(Dictionary<string, object> p)
|
|
266
|
-
{
|
|
267
|
-
string ending = Json.GetString(p, "turnToken");
|
|
268
|
-
if (ending != null) ClearInterrupt(ending);
|
|
269
|
-
lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
|
|
270
|
-
_interruptedTurn = null;
|
|
271
|
-
StopOverlay();
|
|
272
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
273
|
-
r["ended"] = true;
|
|
274
|
-
r["approvedApps"] = (long)_approvedApps.Count;
|
|
275
|
-
return r;
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
public Dictionary<string, object> InvalidateState(Dictionary<string, object> p)
|
|
279
|
-
{
|
|
280
|
-
lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
|
|
281
|
-
StopOverlay();
|
|
282
|
-
return new Dictionary<string, object> { { "invalidated", true } };
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
void AssertAppApproved(IntPtr h)
|
|
286
|
-
{
|
|
287
|
-
if (!_requireAppApproval) return;
|
|
288
|
-
uint pid;
|
|
289
|
-
GetWindowThreadProcessId(h, out pid);
|
|
290
|
-
string id = ProcessPathOf(pid);
|
|
291
|
-
lock (_sync) { if (_approvedApps.Contains(id)) return; }
|
|
292
|
-
throw new ToolError("app_not_approved", "the Computer Use policy has not approved '" + id + "' yet. Approve the app through the DSH approval prompt before acting on its windows.");
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// ---- audio recording (WASAPI loopback) ----
|
|
296
|
-
public Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p)
|
|
297
|
-
{
|
|
298
|
-
bool envEnabled = Environment.GetEnvironmentVariable("DSH_CU_AUDIO") == "1";
|
|
299
|
-
bool force = Json.GetBool(p, "force", false);
|
|
300
|
-
// Codex treats computer audio as its own app ("computer-audio") and asks for a
|
|
301
|
-
// separate approval before recording; the plugin passes that here.
|
|
302
|
-
bool audioApproved = Json.GetString(p, "approvedApp") == "computer-audio";
|
|
303
|
-
if (!envEnabled && !force && !audioApproved)
|
|
304
|
-
throw new ToolError("audio_not_enabled", "computer audio recording is not enabled. Approve it (app \"computer-audio\") or set DSH_CU_AUDIO=1.");
|
|
305
|
-
if (_audioProcess != null && !_audioProcess.HasExited)
|
|
306
|
-
throw new ToolError("audio_running", "an audio recording is already running; stop it first");
|
|
307
|
-
|
|
308
|
-
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
|
|
309
|
-
string exe = Path.Combine(dir, "audio", "dsh-cu-audio.exe");
|
|
310
|
-
if (!File.Exists(exe)) throw new ToolError("audio_missing", "audio recorder not found at " + exe);
|
|
311
|
-
|
|
312
|
-
long maxMs = Json.GetLong(p, "maxDurationMs", 60000);
|
|
313
|
-
if (maxMs < 500) maxMs = 500;
|
|
314
|
-
if (maxMs > 300000) maxMs = 300000; // Codex caps a single recording at 5 minutes
|
|
315
|
-
_audioPath = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-" + Guid.NewGuid().ToString("N") + ".wav");
|
|
316
|
-
_audioStopFile = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-stop-" + Guid.NewGuid().ToString("N") + ".flag");
|
|
317
|
-
try { if (File.Exists(_audioStopFile)) File.Delete(_audioStopFile); } catch { }
|
|
318
|
-
|
|
319
|
-
ProcessStartInfo psi = new ProcessStartInfo(exe, "--out \"" + _audioPath + "\" --max-ms " + maxMs + " --stop-file \"" + _audioStopFile + "\"");
|
|
320
|
-
psi.UseShellExecute = false;
|
|
321
|
-
psi.CreateNoWindow = true;
|
|
322
|
-
psi.RedirectStandardOutput = true;
|
|
323
|
-
psi.RedirectStandardError = true;
|
|
324
|
-
_audioProcess = Process.Start(psi);
|
|
325
|
-
System.Threading.Thread.Sleep(400);
|
|
326
|
-
if (_audioProcess.HasExited)
|
|
327
|
-
throw new ToolError("audio_not_started", "computer audio recording did not start in time (the recorder exited immediately)");
|
|
328
|
-
|
|
329
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
330
|
-
r["started"] = true;
|
|
331
|
-
r["filepath"] = _audioPath;
|
|
332
|
-
r["maxDurationMs"] = maxMs;
|
|
333
|
-
return r;
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
public Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p)
|
|
337
|
-
{
|
|
338
|
-
if (_audioProcess == null) throw new ToolError("audio_not_started", "no audio recording is running");
|
|
339
|
-
try { if (_audioStopFile != null) File.WriteAllText(_audioStopFile, "stop"); } catch { }
|
|
340
|
-
try { _audioProcess.WaitForExit(8000); } catch { }
|
|
341
|
-
try { if (_audioProcess != null && !_audioProcess.HasExited) _audioProcess.Kill(); } catch { }
|
|
342
|
-
|
|
343
|
-
string path = _audioPath;
|
|
344
|
-
_audioProcess = null;
|
|
345
|
-
_audioStopFile = null;
|
|
346
|
-
_audioPath = null;
|
|
347
|
-
|
|
348
|
-
byte[] audio = new byte[0];
|
|
349
|
-
try { if (path != null && File.Exists(path)) audio = File.ReadAllBytes(path); } catch { }
|
|
350
|
-
|
|
351
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
352
|
-
r["filepath"] = path;
|
|
353
|
-
// Codex returns the raw WAV bytes plus a data URL; base64 is the JSON-safe form.
|
|
354
|
-
r["bytes"] = Convert.ToBase64String(audio);
|
|
355
|
-
r["byteLength"] = (long)audio.Length;
|
|
356
|
-
r["data_url"] = audio.Length > 44 ? "data:audio/wav;base64," + Convert.ToBase64String(audio) : "";
|
|
357
|
-
r["ok"] = audio.Length > 44;
|
|
358
|
-
if (audio.Length <= 44) throw new ToolError("audio_not_recorded", "computer audio recording produced no samples");
|
|
359
|
-
return r;
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// ---- in-use indicator ----
|
|
363
|
-
static string OverlayExePath()
|
|
364
|
-
{
|
|
365
|
-
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location);
|
|
366
|
-
return Path.Combine(dir ?? ".", "dsh-cu-overlay.exe");
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
public Dictionary<string, object> OverlayShow(Dictionary<string, object> p)
|
|
370
|
-
{
|
|
371
|
-
string text = Json.GetString(p, "text");
|
|
372
|
-
if (!string.IsNullOrEmpty(text)) _overlayText = text;
|
|
373
|
-
string hint = Json.GetString(p, "hint");
|
|
374
|
-
if (!string.IsNullOrEmpty(hint)) _overlayHint = hint;
|
|
375
|
-
string accent = Json.GetString(p, "accent");
|
|
376
|
-
if (!string.IsNullOrEmpty(accent)) _overlayAccent = accent;
|
|
377
|
-
EnsureOverlay(Json.GetLong(p, "ttlMs", 20000));
|
|
378
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
379
|
-
r["visible"] = true;
|
|
380
|
-
r["cancelFile"] = _overlayCancelFile;
|
|
381
|
-
return r;
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
public Dictionary<string, object> OverlayHide(Dictionary<string, object> p)
|
|
385
|
-
{
|
|
386
|
-
StopOverlay();
|
|
387
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
388
|
-
r["visible"] = false;
|
|
389
|
-
return r;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
void EnsureOverlay(long ttlMs)
|
|
393
|
-
{
|
|
394
|
-
string exe = OverlayExePath();
|
|
395
|
-
if (!File.Exists(exe)) return;
|
|
396
|
-
string mode = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR");
|
|
397
|
-
if (mode != null && mode.Equals("off", StringComparison.OrdinalIgnoreCase)) { StopOverlay(); return; }
|
|
398
|
-
string ttlEnv = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR_TTL_MS");
|
|
399
|
-
long parsed;
|
|
400
|
-
if (ttlEnv != null && long.TryParse(ttlEnv, out parsed) && parsed >= 1000) ttlMs = parsed;
|
|
401
|
-
if (_overlay != null && !_overlay.HasExited && DateTime.UtcNow < _overlayUntil)
|
|
402
|
-
{
|
|
403
|
-
_overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
|
|
404
|
-
return;
|
|
405
|
-
}
|
|
406
|
-
StopOverlay();
|
|
407
|
-
_overlayCancelFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cancel-" + Guid.NewGuid().ToString("N") + ".flag");
|
|
408
|
-
if (_cursorFile == null) _cursorFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cursor-" + Guid.NewGuid().ToString("N").Substring(0, 8) + ".flag");
|
|
409
|
-
try { if (File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
|
|
410
|
-
StringBuilder args = new StringBuilder();
|
|
411
|
-
args.Append("--ttl-ms ").Append(ttlMs);
|
|
412
|
-
args.Append(" --cancel-file \"").Append(_overlayCancelFile).Append('"');
|
|
413
|
-
args.Append(" --text \"").Append(_overlayText).Append('"');
|
|
414
|
-
args.Append(" --hint \"").Append(_overlayHint).Append('"');
|
|
415
|
-
args.Append(" --owner-pid ").Append(Process.GetCurrentProcess().Id);
|
|
416
|
-
args.Append(" --accent \"").Append(_overlayAccent).Append('"');
|
|
417
|
-
if (_cursorFile != null) args.Append(" --cursor-file \"").Append(_cursorFile).Append('"');
|
|
418
|
-
// Suppression is the default (the virtual cursor stands in for the real one);
|
|
419
|
-
// DSH_CU_KEEP_CURSOR=1 keeps the real pointer visible instead.
|
|
420
|
-
if (Environment.GetEnvironmentVariable("DSH_CU_KEEP_CURSOR") == "1") args.Append(" --no-suppress-cursor");
|
|
421
|
-
try
|
|
422
|
-
{
|
|
423
|
-
ProcessStartInfo psi = new ProcessStartInfo(exe, args.ToString());
|
|
424
|
-
psi.UseShellExecute = false;
|
|
425
|
-
_overlay = Process.Start(psi);
|
|
426
|
-
_overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
|
|
427
|
-
// Codex runs a dedicated system-cursor-manager process that restores the
|
|
428
|
-
// pointer the moment its parent dies. This timer gives the same guarantee
|
|
429
|
-
// without a second exe: if the overlay vanishes with cursors hidden, the
|
|
430
|
-
// pointer comes back within half a second, with no request needed.
|
|
431
|
-
if (_cursorWatchdog == null)
|
|
432
|
-
_cursorWatchdog = new System.Threading.Timer(delegate { try { Maintenance(); } catch { } }, null, 500, 500);
|
|
433
|
-
}
|
|
434
|
-
catch { _overlay = null; }
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
void StopOverlay()
|
|
438
|
-
{
|
|
439
|
-
try { if (_overlay != null && !_overlay.HasExited) _overlay.Kill(); } catch { }
|
|
440
|
-
// Kill() skips the overlay's exit handlers, which is where it would restore the
|
|
441
|
-
// system cursors. Restore them from here so a terminated overlay can never leave
|
|
442
|
-
// the user without a pointer.
|
|
443
|
-
RestoreCursors();
|
|
444
|
-
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
445
|
-
try { if (_overlayCancelFile != null && File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
|
|
446
|
-
_overlay = null;
|
|
447
|
-
_overlayUntil = DateTime.MinValue;
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/// Esc on the overlay drops a cancel file; honour it before any other input.
|
|
451
|
-
const string EscMessage = "Computer Use was stopped by the user with the physical Escape key. Stop your work, do not call further Computer Use tools in this turn, and send a final message noting that the user stopped Computer Use.";
|
|
452
|
-
|
|
453
|
-
void CheckCancel(Dictionary<string, object> p)
|
|
454
|
-
{
|
|
455
|
-
Budget.Check();
|
|
456
|
-
string token = p == null ? null : Json.GetString(p, "turnToken");
|
|
457
|
-
if (token != null && (token == _interruptedTurn || InterruptMarkerExists(token)))
|
|
458
|
-
throw new ToolError("turn_stopped", EscMessage);
|
|
459
|
-
bool overlayExited = _overlay != null && _overlay.HasExited && _overlay.ExitCode == 130;
|
|
460
|
-
bool cancelFile = _overlayCancelFile != null && File.Exists(_overlayCancelFile);
|
|
461
|
-
if (!overlayExited && !cancelFile) return;
|
|
462
|
-
if (cancelFile) { try { File.Delete(_overlayCancelFile); } catch { } }
|
|
463
|
-
_interruptedTurn = token;
|
|
464
|
-
PersistInterrupt(token);
|
|
465
|
-
StopOverlay();
|
|
466
|
-
Program.RequestedExitCode = 130; // Codex exits 130 when the user hits Esc
|
|
467
|
-
throw new ToolError("cancelled_by_user", EscMessage);
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
const int ProtocolVersion = 2;
|
|
471
|
-
|
|
472
|
-
static string InterruptMarkerPath(string token)
|
|
473
|
-
{
|
|
474
|
-
string root = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "interrupts");
|
|
475
|
-
StringBuilder safe = new StringBuilder();
|
|
476
|
-
if (token != null)
|
|
477
|
-
{
|
|
478
|
-
for (int i = 0; i < token.Length && safe.Length < 64; i++)
|
|
479
|
-
{
|
|
480
|
-
char ch = token[i];
|
|
481
|
-
if (char.IsLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.') safe.Append(ch);
|
|
482
|
-
else if (ch == ':') safe.Append('-');
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
if (safe.Length == 0) safe.Append("turn");
|
|
486
|
-
return Path.Combine(root, safe.ToString() + ".flag");
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
static void PersistInterrupt(string token)
|
|
490
|
-
{
|
|
491
|
-
try
|
|
492
|
-
{
|
|
493
|
-
string path = InterruptMarkerPath(token);
|
|
494
|
-
Directory.CreateDirectory(Path.GetDirectoryName(path));
|
|
495
|
-
File.WriteAllText(path, DateTime.UtcNow.ToString("o"));
|
|
496
|
-
}
|
|
497
|
-
catch { }
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
static bool InterruptMarkerExists(string token)
|
|
501
|
-
{
|
|
502
|
-
try { return File.Exists(InterruptMarkerPath(token)); } catch { return false; }
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
static void ClearInterrupt(string token)
|
|
506
|
-
{
|
|
507
|
-
try { File.Delete(InterruptMarkerPath(token)); } catch { }
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
/// Publish the pointer target for the virtual cursor. The overlay reads this
|
|
511
|
-
/// file; a missing file simply means "no cursor to draw".
|
|
512
|
-
void MarkCursor(int x, int y)
|
|
513
|
-
{
|
|
514
|
-
if (_cursorFile == null) return;
|
|
515
|
-
try { File.WriteAllText(_cursorFile, x.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + y.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture)); } catch { }
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
/// Codex reports screenLocked instead of failing with an opaque automation error.
|
|
519
|
-
static void AssertUnlocked()
|
|
520
|
-
{
|
|
521
|
-
IntPtr fg = GetForegroundWindow();
|
|
522
|
-
if (fg == IntPtr.Zero) return;
|
|
523
|
-
uint pid;
|
|
524
|
-
GetWindowThreadProcessId(fg, out pid);
|
|
525
|
-
string proc = "";
|
|
526
|
-
try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
|
|
527
|
-
if (proc.Equals("LockApp", StringComparison.OrdinalIgnoreCase) || proc.Equals("LogonUI", StringComparison.OrdinalIgnoreCase))
|
|
528
|
-
throw new ToolError("screen_locked", "the Windows desktop is locked (LockApp is in the foreground), so Computer Use cannot interact with it. Ask the user to unlock the desktop and stop trying until they do.");
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
// ---- helpers ----
|
|
532
|
-
static string TitleOf(IntPtr h)
|
|
533
|
-
{
|
|
534
|
-
StringBuilder sb = new StringBuilder(1024);
|
|
535
|
-
GetWindowTextW(h, sb, sb.Capacity);
|
|
536
|
-
return sb.ToString();
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
static string ClassOf(IntPtr h)
|
|
540
|
-
{
|
|
541
|
-
StringBuilder sb = new StringBuilder(256);
|
|
542
|
-
GetClassNameW(h, sb, sb.Capacity);
|
|
543
|
-
return sb.ToString();
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
static RECT BoundsOf(IntPtr h)
|
|
547
|
-
{
|
|
548
|
-
RECT r;
|
|
549
|
-
try
|
|
550
|
-
{
|
|
551
|
-
if (DwmGetWindowAttribute(h, DWMWA_EXTENDED_FRAME_BOUNDS, out r, Marshal.SizeOf(typeof(RECT))) == 0 && r.Right > r.Left)
|
|
552
|
-
return r;
|
|
553
|
-
}
|
|
554
|
-
catch { }
|
|
555
|
-
GetWindowRect(h, out r);
|
|
556
|
-
return r;
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
static Dictionary<string, object> RectDict(int x, int y, int w, int h)
|
|
560
|
-
{
|
|
561
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
562
|
-
d["x"] = x; d["y"] = y; d["width"] = w; d["height"] = h;
|
|
563
|
-
return d;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
static string ProcessPathOf(uint pid)
|
|
567
|
-
{
|
|
568
|
-
try
|
|
569
|
-
{
|
|
570
|
-
Process p = Process.GetProcessById((int)pid);
|
|
571
|
-
try { return p.MainModule.FileName; }
|
|
572
|
-
catch { return p.ProcessName; }
|
|
573
|
-
}
|
|
574
|
-
catch { return "unknown"; }
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
static string ProcessNameOf(uint pid)
|
|
578
|
-
{
|
|
579
|
-
try { return Process.GetProcessById((int)pid).ProcessName; }
|
|
580
|
-
catch { return "unknown"; }
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
Dictionary<string, object> AppOf(uint pid)
|
|
584
|
-
{
|
|
585
|
-
Dictionary<string, object> app = new Dictionary<string, object>();
|
|
586
|
-
string full = ProcessPathOf(pid);
|
|
587
|
-
app["id"] = full;
|
|
588
|
-
app["name"] = Path.GetFileNameWithoutExtension(full);
|
|
589
|
-
app["displayName"] = ProcessNameOf(pid);
|
|
590
|
-
app["pid"] = (long)pid;
|
|
591
|
-
try
|
|
592
|
-
{
|
|
593
|
-
Process proc = Process.GetProcessById((int)pid);
|
|
594
|
-
app["lastUsedDate"] = proc.StartTime.ToUniversalTime().ToString("o");
|
|
595
|
-
}
|
|
596
|
-
catch { }
|
|
597
|
-
return app;
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
Dictionary<string, object> WindowDict(IntPtr h, uint pid, int z)
|
|
601
|
-
{
|
|
602
|
-
RECT r = BoundsOf(h);
|
|
603
|
-
Dictionary<string, object> w = new Dictionary<string, object>();
|
|
604
|
-
w["id"] = (long)h.ToInt64();
|
|
605
|
-
w["title"] = TitleOf(h);
|
|
606
|
-
w["app"] = AppOf(pid);
|
|
607
|
-
w["bounds"] = RectDict(r.Left, r.Top, r.Right - r.Left, r.Bottom - r.Top);
|
|
608
|
-
w["zIndex"] = z;
|
|
609
|
-
w["className"] = ClassOf(h);
|
|
610
|
-
return w;
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
List<IntPtr> EnumerateWindows(bool visibleOnly)
|
|
614
|
-
{
|
|
615
|
-
List<IntPtr> list = new List<IntPtr>();
|
|
616
|
-
EnumWindows(delegate(IntPtr h, IntPtr lp)
|
|
617
|
-
{
|
|
618
|
-
if (!IsWindow(h)) return true;
|
|
619
|
-
if (visibleOnly && !IsWindowVisible(h)) return true;
|
|
620
|
-
uint pid;
|
|
621
|
-
GetWindowThreadProcessId(h, out pid);
|
|
622
|
-
if (pid == 0) return true;
|
|
623
|
-
RECT r;
|
|
624
|
-
GetWindowRect(h, out r);
|
|
625
|
-
if (r.Right - r.Left < 80 || r.Bottom - r.Top < 60) return true;
|
|
626
|
-
string title = TitleOf(h);
|
|
627
|
-
if (title.Length == 0) return true;
|
|
628
|
-
list.Add(h);
|
|
629
|
-
return true;
|
|
630
|
-
}, IntPtr.Zero);
|
|
631
|
-
return list;
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
// ---- methods ----
|
|
635
|
-
public Dictionary<string, object> Ping(Dictionary<string, object> p)
|
|
636
|
-
{
|
|
637
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
638
|
-
d["platform"] = PlatformName;
|
|
639
|
-
d["pid"] = (long)Process.GetCurrentProcess().Id;
|
|
640
|
-
d["overlayPid"] = _overlay != null && !_overlay.HasExited ? (object)(long)_overlay.Id : null;
|
|
641
|
-
d["ok"] = true;
|
|
642
|
-
d["uia"] = true;
|
|
643
|
-
return d;
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
/// Run cheaply before every request: if a previous overlay died while it was hiding
|
|
647
|
-
/// the system cursors, put them back. A blank desktop cursor is worse than a stray
|
|
648
|
-
/// visible one, so this never assumes the overlay cleaned up after itself.
|
|
649
|
-
public void Maintenance()
|
|
650
|
-
{
|
|
651
|
-
try
|
|
652
|
-
{
|
|
653
|
-
string flag = OverlayCursorFlagPath();
|
|
654
|
-
if (!File.Exists(flag)) return;
|
|
655
|
-
int pid;
|
|
656
|
-
if (!int.TryParse(File.ReadAllText(flag).Trim(), out pid)) { RestoreCursors(); return; }
|
|
657
|
-
bool alive;
|
|
658
|
-
try { Process.GetProcessById(pid); alive = true; } catch { alive = false; }
|
|
659
|
-
if (!alive) { RestoreCursors(); try { File.Delete(flag); } catch { } }
|
|
660
|
-
}
|
|
661
|
-
catch { }
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
static string OverlayCursorFlagPath()
|
|
665
|
-
{
|
|
666
|
-
return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "cursor-hidden.flag");
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
static void RestoreCursors()
|
|
670
|
-
{
|
|
671
|
-
try { SystemParametersInfo(0x0057, 0, IntPtr.Zero, 0); } catch { }
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
public Dictionary<string, object> RestoreCursor(Dictionary<string, object> p)
|
|
675
|
-
{
|
|
676
|
-
RestoreCursors();
|
|
677
|
-
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
678
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
679
|
-
r["restored"] = true;
|
|
680
|
-
return r;
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
public Dictionary<string, object> Hello(Dictionary<string, object> p)
|
|
684
|
-
{
|
|
685
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
686
|
-
d["ok"] = true;
|
|
687
|
-
d["name"] = "dsh-computer-use";
|
|
688
|
-
d["protocolVersion"] = ProtocolVersion;
|
|
689
|
-
d["platform"] = PlatformName;
|
|
690
|
-
List<object> caps = new List<object>();
|
|
691
|
-
foreach (string c in new string[] {
|
|
692
|
-
"window2", "accessibility_tree", "multi_screenshot", "wgc_capture",
|
|
693
|
-
"virtual_cursor", "audio_recording", "turn_interrupt", "app_approval",
|
|
694
|
-
"screenshot_ids", "budgeted_capture" }) caps.Add(c);
|
|
695
|
-
d["capabilities"] = caps;
|
|
696
|
-
return d;
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
public Dictionary<string, object> Close(Dictionary<string, object> p)
|
|
700
|
-
{
|
|
701
|
-
Program.RequestedExitCode = 0;
|
|
702
|
-
Program.CloseRequested = true;
|
|
703
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
704
|
-
d["closed"] = true;
|
|
705
|
-
return d;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
public Dictionary<string, object> ListWindows(Dictionary<string, object> p)
|
|
709
|
-
{
|
|
710
|
-
List<object> outList = new List<object>();
|
|
711
|
-
List<IntPtr> hs = EnumerateWindows(true);
|
|
712
|
-
for (int i = 0; i < hs.Count; i++)
|
|
713
|
-
{
|
|
714
|
-
uint pid;
|
|
715
|
-
GetWindowThreadProcessId(hs[i], out pid);
|
|
716
|
-
outList.Add(WindowDict(hs[i], pid, i));
|
|
717
|
-
}
|
|
718
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
719
|
-
d["windows"] = outList;
|
|
720
|
-
d["count"] = outList.Count;
|
|
721
|
-
return d;
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
public Dictionary<string, object> ListApps(Dictionary<string, object> p)
|
|
725
|
-
{
|
|
726
|
-
Dictionary<string, List<object>> grouped = new Dictionary<string, List<object>>();
|
|
727
|
-
Dictionary<string, Dictionary<string, object>> appInfo = new Dictionary<string, Dictionary<string, object>>();
|
|
728
|
-
List<IntPtr> hs = EnumerateWindows(true);
|
|
729
|
-
for (int i = 0; i < hs.Count; i++)
|
|
730
|
-
{
|
|
731
|
-
uint pid;
|
|
732
|
-
GetWindowThreadProcessId(hs[i], out pid);
|
|
733
|
-
Dictionary<string, object> app = AppOf(pid);
|
|
734
|
-
string key = Convert.ToString(app["id"]);
|
|
735
|
-
if (!grouped.ContainsKey(key)) { grouped[key] = new List<object>(); appInfo[key] = app; }
|
|
736
|
-
grouped[key].Add(WindowDict(hs[i], pid, i));
|
|
737
|
-
}
|
|
738
|
-
List<object> apps = new List<object>();
|
|
739
|
-
foreach (KeyValuePair<string, List<object>> kv in grouped)
|
|
740
|
-
{
|
|
741
|
-
Dictionary<string, object> a = appInfo[kv.Key];
|
|
742
|
-
a["isRunning"] = true;
|
|
743
|
-
try
|
|
744
|
-
{
|
|
745
|
-
IntPtr sample = IntPtr.Zero;
|
|
746
|
-
List<IntPtr> windowsOfApp = new List<IntPtr>();
|
|
747
|
-
for (int w = 0; w < hs.Count; w++)
|
|
748
|
-
{
|
|
749
|
-
uint wpid;
|
|
750
|
-
GetWindowThreadProcessId(hs[w], out wpid);
|
|
751
|
-
if (Convert.ToString(AppOf(wpid)["id"]) == kv.Key) { sample = hs[w]; break; }
|
|
752
|
-
}
|
|
753
|
-
if (sample != IntPtr.Zero)
|
|
754
|
-
{
|
|
755
|
-
uint spid;
|
|
756
|
-
GetWindowThreadProcessId(sample, out spid);
|
|
757
|
-
a["lastUsedDate"] = Process.GetProcessById((int)spid).StartTime.ToUniversalTime().ToString("o");
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
catch { }
|
|
761
|
-
a["windows"] = kv.Value;
|
|
762
|
-
apps.Add(a);
|
|
763
|
-
}
|
|
764
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
765
|
-
d["apps"] = apps;
|
|
766
|
-
d["count"] = apps.Count;
|
|
767
|
-
return d;
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
public Dictionary<string, object> GetWindow(Dictionary<string, object> p)
|
|
771
|
-
{
|
|
772
|
-
IntPtr h = ResolveWindow(p);
|
|
773
|
-
uint pid;
|
|
774
|
-
GetWindowThreadProcessId(h, out pid);
|
|
775
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
776
|
-
d["window"] = WindowDict(h, pid, 0);
|
|
777
|
-
return d;
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
public Dictionary<string, object> ActivateWindow(Dictionary<string, object> p)
|
|
781
|
-
{
|
|
782
|
-
IntPtr h = ResolveWindow(p);
|
|
783
|
-
Activate(h);
|
|
784
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
785
|
-
d["activated"] = true;
|
|
786
|
-
d["window"] = WindowDict(h, PidOf(h), 0);
|
|
787
|
-
return d;
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
static uint PidOf(IntPtr h) { uint pid; GetWindowThreadProcessId(h, out pid); return pid; }
|
|
791
|
-
|
|
792
|
-
/// Bring a window to the foreground and PROVE it got there. Input is never
|
|
793
|
-
/// injected into a window that is not foreground: SendInput targets whatever
|
|
794
|
-
/// holds focus, so an unproven activation would type into the user's window.
|
|
795
|
-
static void Activate(IntPtr h)
|
|
796
|
-
{
|
|
797
|
-
if (!EnsureForeground(h))
|
|
798
|
-
throw new ToolError("focus_failed", "could not bring window " + h.ToInt64() + " to the foreground, so the input was refused instead of landing in another window. " + DescribeForeground());
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
static bool EnsureForeground(IntPtr h)
|
|
802
|
-
{
|
|
803
|
-
if (!IsWindow(h)) throw new ToolError("window_gone", "window " + h.ToInt64() + " no longer exists; call list_windows again");
|
|
804
|
-
if (!IsWindowEnabled(h)) throw new ToolError("window_disabled", "window " + h.ToInt64() + " is disabled (a modal dialog may be blocking it)");
|
|
805
|
-
if (IsIconic(h)) { ShowWindow(h, 9); System.Threading.Thread.Sleep(180); }
|
|
806
|
-
if (GetForegroundWindow() == h) return true;
|
|
807
|
-
|
|
808
|
-
uint fgThread = GetWindowThreadProcessId(GetForegroundWindow(), IntPtr.Zero);
|
|
809
|
-
uint thisThread = GetCurrentThreadId();
|
|
810
|
-
bool attached = false;
|
|
811
|
-
try
|
|
812
|
-
{
|
|
813
|
-
if (fgThread != 0 && fgThread != thisThread) attached = AttachThreadInput(thisThread, fgThread, true);
|
|
814
|
-
ShowWindow(h, 5); // SW_SHOW
|
|
815
|
-
BringWindowToTop(h);
|
|
816
|
-
SetForegroundWindow(h);
|
|
817
|
-
SetFocus(h);
|
|
818
|
-
}
|
|
819
|
-
finally { if (attached) AttachThreadInput(thisThread, fgThread, false); }
|
|
820
|
-
|
|
821
|
-
if (WaitForeground(h, 10)) return true;
|
|
822
|
-
|
|
823
|
-
// Windows refuses foreground changes from a background process. The
|
|
824
|
-
// documented-looking escape hatches, in order of intrusiveness:
|
|
825
|
-
// 1. SwitchToThisWindow (alt-tab semantics, no key injection)
|
|
826
|
-
// 2. a bare Alt press, which lifts the foreground lock for this call
|
|
827
|
-
try { SwitchToThisWindow(h, true); } catch { }
|
|
828
|
-
if (WaitForeground(h, 10)) return true;
|
|
829
|
-
|
|
830
|
-
keybd_event(0x12, 0, 0, UIntPtr.Zero); // VK_MENU down
|
|
831
|
-
keybd_event(0x12, 0, 0x0002, UIntPtr.Zero); // VK_MENU up
|
|
832
|
-
SetForegroundWindow(h);
|
|
833
|
-
if (WaitForeground(h, 12)) return true;
|
|
834
|
-
return false;
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
/// Name the process that currently owns focus: the usual reason a
|
|
838
|
-
/// background host cannot steal it is an always-on-top utility or an
|
|
839
|
-
/// elevated window.
|
|
840
|
-
static string DescribeForeground()
|
|
841
|
-
{
|
|
842
|
-
IntPtr fg = GetForegroundWindow();
|
|
843
|
-
if (fg == IntPtr.Zero) return "No window currently holds the foreground.";
|
|
844
|
-
uint pid;
|
|
845
|
-
GetWindowThreadProcessId(fg, out pid);
|
|
846
|
-
string proc = "unknown";
|
|
847
|
-
try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
|
|
848
|
-
string title = TitleOf(fg);
|
|
849
|
-
string cls = ClassOf(fg);
|
|
850
|
-
return "Foreground is held by " + proc + " (pid " + pid + ", class " + cls + (title.Length > 0 ? ", title \"" + Trim(title, 60) + "\"" : "") + "). Bring the target window forward yourself, or close/pause the utility that keeps stealing focus.";
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
static bool WaitForeground(IntPtr h, int attempts)
|
|
854
|
-
{
|
|
855
|
-
for (int i = 0; i < attempts; i++)
|
|
856
|
-
{
|
|
857
|
-
if (GetForegroundWindow() == h) { System.Threading.Thread.Sleep(70); return true; }
|
|
858
|
-
System.Threading.Thread.Sleep(40);
|
|
859
|
-
}
|
|
860
|
-
return GetForegroundWindow() == h;
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
static void Slow() { System.Threading.Thread.Sleep(120); }
|
|
864
|
-
|
|
865
|
-
public Dictionary<string, object> LaunchApp(Dictionary<string, object> p)
|
|
866
|
-
{
|
|
867
|
-
string app = Json.GetString(p, "app");
|
|
868
|
-
if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "launch_app requires 'app'");
|
|
869
|
-
// Resolution order matches the API contract: an explicit .exe path first,
|
|
870
|
-
// then a packaged-app id (AUMID) through shell:AppsFolder, then a shell URI
|
|
871
|
-
// such as ms-settings:. A bare unknown string is tried verbatim last.
|
|
872
|
-
string resolved = app;
|
|
873
|
-
if (!File.Exists(app) && app.IndexOf('\\') < 0 && !app.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) && app.IndexOf(':') < 0)
|
|
874
|
-
resolved = "shell:AppsFolder\\" + app;
|
|
875
|
-
ProcessStartInfo psi = new ProcessStartInfo();
|
|
876
|
-
psi.FileName = resolved;
|
|
877
|
-
psi.UseShellExecute = true;
|
|
878
|
-
List<object> args = Json.Get(p, "args") as List<object>;
|
|
879
|
-
if (args != null && args.Count > 0)
|
|
880
|
-
{
|
|
881
|
-
StringBuilder argLine = new StringBuilder();
|
|
882
|
-
for (int i = 0; i < args.Count; i++)
|
|
883
|
-
{
|
|
884
|
-
if (i > 0) argLine.Append(' ');
|
|
885
|
-
string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture) ?? "";
|
|
886
|
-
// A quote or newline would let a caller smuggle extra arguments past the
|
|
887
|
-
// single quoting rule below, so refuse instead of guessing.
|
|
888
|
-
if (a.IndexOf('"') >= 0 || a.IndexOf('\n') >= 0 || a.IndexOf('\r') >= 0)
|
|
889
|
-
throw new ToolError("bad_request", "launch_app argument " + i + " contains a quote or newline; pass a single plain argument");
|
|
890
|
-
if (a.Length == 0 || a.IndexOf(' ') >= 0 || a.IndexOf('\t') >= 0)
|
|
891
|
-
argLine.Append('"').Append(a).Append('"');
|
|
892
|
-
else argLine.Append(a);
|
|
893
|
-
}
|
|
894
|
-
psi.Arguments = argLine.ToString();
|
|
895
|
-
}
|
|
896
|
-
Process proc = Process.Start(psi);
|
|
897
|
-
System.Threading.Thread.Sleep(900);
|
|
898
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
899
|
-
d["launched"] = true;
|
|
900
|
-
d["app"] = app;
|
|
901
|
-
try { d["pid"] = (long)proc.Id; } catch { }
|
|
902
|
-
return d;
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
IntPtr ResolveWindow(Dictionary<string, object> p)
|
|
906
|
-
{
|
|
907
|
-
Dictionary<string, object> w = Json.AsObject(Json.Get(p, "window"));
|
|
908
|
-
long id = Json.GetLong(w, "id", 0);
|
|
909
|
-
if (id == 0) id = Json.GetLong(p, "windowId", 0);
|
|
910
|
-
if (id == 0) throw new ToolError("bad_request", "a 'window' object with a numeric 'id' is required (use list_windows / list_apps)");
|
|
911
|
-
IntPtr h = new IntPtr(id);
|
|
912
|
-
if (!IsWindow(h)) throw new ToolError("window_gone", "window " + id + " no longer exists; call list_windows again");
|
|
913
|
-
return h;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// ---- observation ----
|
|
917
|
-
public Dictionary<string, object> GetWindowState(Dictionary<string, object> p)
|
|
918
|
-
{
|
|
919
|
-
IntPtr h = ResolveWindow(p);
|
|
920
|
-
AssertAppApproved(h);
|
|
921
|
-
bool includeText = Json.GetBool(p, "include_text", false);
|
|
922
|
-
bool includeShot = Json.GetBool(p, "include_screenshot", false);
|
|
923
|
-
long maxElements = Json.GetLong(p, "maxElements", 260);
|
|
924
|
-
long ttl = Json.GetLong(p, "ttlMs", _snapshotTtlMs);
|
|
925
|
-
|
|
926
|
-
uint pid;
|
|
927
|
-
GetWindowThreadProcessId(h, out pid);
|
|
928
|
-
Dictionary<string, object> result = new Dictionary<string, object>();
|
|
929
|
-
result["window"] = WindowDict(h, pid, 0);
|
|
930
|
-
|
|
931
|
-
// Codex parity: accessibility is null unless include_text was asked for.
|
|
932
|
-
result["accessibility"] = includeText ? (object)BuildAccessibility(h, maxElements, (int)ttl) : null;
|
|
933
|
-
bool blank;
|
|
934
|
-
result["screenshots"] = includeShot ? (object)CaptureScreenshots(h, out blank) : new List<object>();
|
|
935
|
-
return result;
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
Dictionary<string, object> BuildAccessibility(IntPtr h, long maxElements, int ttlMs)
|
|
939
|
-
{
|
|
940
|
-
Snapshot snap = new Snapshot();
|
|
941
|
-
snap.WindowId = h.ToInt64();
|
|
942
|
-
snap.At = DateTime.UtcNow;
|
|
943
|
-
|
|
944
|
-
StringBuilder tree = new StringBuilder();
|
|
945
|
-
int index = 0;
|
|
946
|
-
AutomationElement root = null;
|
|
947
|
-
try { root = AutomationElement.FromHandle(h); }
|
|
948
|
-
catch (Exception ex) { throw new ToolError("uia_unavailable", "UI Automation cannot read this window: " + ex.Message); }
|
|
949
|
-
|
|
950
|
-
string focused = null;
|
|
951
|
-
Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
|
|
952
|
-
|
|
953
|
-
if (index == 0)
|
|
954
|
-
{
|
|
955
|
-
// Electron/Canvas apps may not expose a tree until accessibility is requested;
|
|
956
|
-
// retry once after a short delay.
|
|
957
|
-
System.Threading.Thread.Sleep(180);
|
|
958
|
-
try { root = AutomationElement.FromHandle(h); } catch { }
|
|
959
|
-
Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
lock (_sync) { _snapshots[snap.WindowId] = snap; }
|
|
963
|
-
|
|
964
|
-
Dictionary<string, object> state = new Dictionary<string, object>();
|
|
965
|
-
state["tree"] = tree.ToString();
|
|
966
|
-
state["elementCount"] = (long)index;
|
|
967
|
-
state["elements"] = snap.Summary;
|
|
968
|
-
|
|
969
|
-
// Structured context Codex exposes beside the tree. Each is best-effort:
|
|
970
|
-
// an app that does not implement the pattern simply omits the field.
|
|
971
|
-
try
|
|
972
|
-
{
|
|
973
|
-
AutomationElement doc = root.FindFirst(TreeScope.Descendants,
|
|
974
|
-
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Document));
|
|
975
|
-
if (doc != null)
|
|
976
|
-
{
|
|
977
|
-
object dp;
|
|
978
|
-
if (doc.TryGetCurrentPattern(TextPattern.Pattern, out dp))
|
|
979
|
-
{
|
|
980
|
-
string body = ((TextPattern)dp).DocumentRange.GetText(8000);
|
|
981
|
-
if (!string.IsNullOrEmpty(body)) state["document_text"] = body;
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
object focusPattern;
|
|
985
|
-
AutomationElement focusedElement = AutomationElement.FocusedElement;
|
|
986
|
-
if (focusedElement != null && focusedElement.TryGetCurrentPattern(TextPattern.Pattern, out focusPattern))
|
|
987
|
-
{
|
|
988
|
-
System.Windows.Automation.Text.TextPatternRange[] ranges = ((TextPattern)focusPattern).GetSelection();
|
|
989
|
-
if (ranges != null && ranges.Length > 0)
|
|
990
|
-
{
|
|
991
|
-
string selected = ranges[0].GetText(4000);
|
|
992
|
-
if (!string.IsNullOrEmpty(selected)) state["selected_text"] = selected;
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
Condition selectable = new PropertyCondition(AutomationElement.IsSelectionItemPatternAvailableProperty, true);
|
|
996
|
-
AutomationElementCollection selectedItems = root.FindAll(TreeScope.Descendants, selectable);
|
|
997
|
-
List<object> selectedLines = new List<object>();
|
|
998
|
-
for (int i = 0; i < selectedItems.Count && selectedLines.Count < 20; i++)
|
|
999
|
-
{
|
|
1000
|
-
try
|
|
1001
|
-
{
|
|
1002
|
-
object sp;
|
|
1003
|
-
if (selectedItems[i].TryGetCurrentPattern(SelectionItemPattern.Pattern, out sp) && ((SelectionItemPattern)sp).Current.IsSelected)
|
|
1004
|
-
selectedLines.Add(SafeString(selectedItems[i].Current.Name) + " (" + SafeRole(selectedItems[i].Current) + ")");
|
|
1005
|
-
}
|
|
1006
|
-
catch { }
|
|
1007
|
-
}
|
|
1008
|
-
if (selectedLines.Count > 0) state["selected_elements"] = selectedLines;
|
|
1009
|
-
}
|
|
1010
|
-
catch { }
|
|
1011
|
-
state["snapshotAt"] = snap.At.ToString("o");
|
|
1012
|
-
state["ttlMs"] = (long)ttlMs;
|
|
1013
|
-
if (focused != null) state["focused_element"] = focused;
|
|
1014
|
-
return state;
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
void Walk(AutomationElement el, int depth, long maxElements, Snapshot snap, StringBuilder sb, ref int index, IntPtr hwnd, ref string focused)
|
|
1018
|
-
{
|
|
1019
|
-
if (el == null) return;
|
|
1020
|
-
AutomationElementCollection kids;
|
|
1021
|
-
try { kids = el.FindAll(TreeScope.Children, Condition.TrueCondition); }
|
|
1022
|
-
catch { return; }
|
|
1023
|
-
|
|
1024
|
-
for (int i = 0; i < kids.Count; i++)
|
|
1025
|
-
{
|
|
1026
|
-
if (index >= maxElements) return;
|
|
1027
|
-
AutomationElement k;
|
|
1028
|
-
try { k = kids[i]; } catch { continue; }
|
|
1029
|
-
AutomationElement.AutomationElementInformation info;
|
|
1030
|
-
bool offscreen;
|
|
1031
|
-
try { info = k.Current; offscreen = info.IsOffscreen; } catch { continue; }
|
|
1032
|
-
if (offscreen) continue;
|
|
1033
|
-
try
|
|
1034
|
-
{
|
|
1035
|
-
|
|
1036
|
-
long idx = index++;
|
|
1037
|
-
snap.Elements[idx] = k;
|
|
1038
|
-
|
|
1039
|
-
string role = SafeRole(info);
|
|
1040
|
-
string name = SafeString(info.Name);
|
|
1041
|
-
System.Windows.Rect rect = System.Windows.Rect.Empty;
|
|
1042
|
-
try { rect = info.BoundingRectangle; } catch { }
|
|
1043
|
-
bool hasRect = !rect.IsEmpty && rect.Width > 0 && rect.Height > 0;
|
|
1044
|
-
|
|
1045
|
-
Dictionary<string, object> summary = new Dictionary<string, object>();
|
|
1046
|
-
summary["index"] = idx;
|
|
1047
|
-
summary["role"] = role;
|
|
1048
|
-
summary["name"] = name;
|
|
1049
|
-
if (hasRect)
|
|
1050
|
-
{
|
|
1051
|
-
summary["x"] = (int)rect.X;
|
|
1052
|
-
summary["y"] = (int)rect.Y;
|
|
1053
|
-
summary["width"] = (int)rect.Width;
|
|
1054
|
-
summary["height"] = (int)rect.Height;
|
|
1055
|
-
}
|
|
1056
|
-
snap.Summary.Add(summary);
|
|
1057
|
-
|
|
1058
|
-
sb.Append('[').Append(idx).Append("] ").Append(role);
|
|
1059
|
-
if (name.Length > 0) sb.Append(" \"").Append(Trim(name, 120)).Append('"');
|
|
1060
|
-
if (hasRect) sb.Append(" @").Append((int)rect.X).Append(',').Append((int)rect.Y).Append(' ').Append((int)rect.Width).Append('x').Append((int)rect.Height);
|
|
1061
|
-
|
|
1062
|
-
List<string> actions = ActionsOf(k);
|
|
1063
|
-
if (actions.Count > 0) sb.Append(" [").Append(string.Join(",", actions.ToArray())).Append(']');
|
|
1064
|
-
|
|
1065
|
-
StringBuilder val = new StringBuilder();
|
|
1066
|
-
try
|
|
1067
|
-
{
|
|
1068
|
-
object vp;
|
|
1069
|
-
if (k.TryGetCurrentPattern(ValuePattern.Pattern, out vp))
|
|
1070
|
-
{
|
|
1071
|
-
string v = ((ValuePattern)vp).Current.Value;
|
|
1072
|
-
if (!string.IsNullOrEmpty(v)) val.Append(" value=\"").Append(Trim(v, 90)).Append('"');
|
|
1073
|
-
}
|
|
1074
|
-
else
|
|
1075
|
-
{
|
|
1076
|
-
object tp;
|
|
1077
|
-
if (k.TryGetCurrentPattern(TextPattern.Pattern, out tp))
|
|
1078
|
-
{
|
|
1079
|
-
string t = ((TextPattern)tp).DocumentRange.GetText(120);
|
|
1080
|
-
if (!string.IsNullOrEmpty(t)) val.Append(" text=\"").Append(Trim(t.Replace("\r", " ").Replace("\n", " "), 90)).Append('"');
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
}
|
|
1084
|
-
catch { }
|
|
1085
|
-
sb.Append(val.ToString());
|
|
1086
|
-
|
|
1087
|
-
try { if (info.HasKeyboardFocus && focused == null) focused = "[" + idx + "] " + role + " \"" + name + "\""; }
|
|
1088
|
-
catch { }
|
|
1089
|
-
|
|
1090
|
-
sb.Append('\n');
|
|
1091
|
-
if (depth < 18) Walk(k, depth + 1, maxElements, snap, sb, ref index, hwnd, ref focused);
|
|
1092
|
-
}
|
|
1093
|
-
catch (Exception) { continue; }
|
|
1094
|
-
}
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
static string SafeRole(AutomationElement.AutomationElementInformation info)
|
|
1098
|
-
{
|
|
1099
|
-
try { return info.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
|
|
1100
|
-
catch { return "Unknown"; }
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
static long ToLong(object v) { try { return Convert.ToInt64(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 0L; } }
|
|
1104
|
-
static double ToDouble(object v) { try { return Convert.ToDouble(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 1.0; } }
|
|
1105
|
-
|
|
1106
|
-
static string SafeString(string s) { return s == null ? "" : s.Replace("\r", " ").Replace("\n", " ").Trim(); }
|
|
1107
|
-
|
|
1108
|
-
static string Trim(string s, int max) { return s.Length <= max ? s : s.Substring(0, max) + "…"; }
|
|
1109
|
-
|
|
1110
|
-
static List<string> ActionsOf(AutomationElement el)
|
|
1111
|
-
{
|
|
1112
|
-
List<string> a = new List<string>();
|
|
1113
|
-
object pattern;
|
|
1114
|
-
try { if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern)) a.Add("Invoke"); } catch { }
|
|
1115
|
-
try { if (el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern)) a.Add("Toggle"); } catch { }
|
|
1116
|
-
try { if (el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern)) a.Add("Select"); } catch { }
|
|
1117
|
-
try { if (el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern)) a.Add("ExpandCollapse"); } catch { }
|
|
1118
|
-
try { if (el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern)) a.Add("Scroll"); } catch { }
|
|
1119
|
-
try { if (el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) a.Add("SetValue"); } catch { }
|
|
1120
|
-
return a;
|
|
1121
|
-
}
|
|
1122
|
-
|
|
1123
|
-
/// Every top-level window that belongs to the same "visual state" as the
|
|
1124
|
-
/// target: the window itself plus owned/transient popups (menus, tooltips,
|
|
1125
|
-
/// combo drop-downs) that a screenshot of the window alone would hide.
|
|
1126
|
-
List<IntPtr> RelatedWindows(IntPtr target)
|
|
1127
|
-
{
|
|
1128
|
-
RECT main = BoundsOf(target);
|
|
1129
|
-
uint targetPid;
|
|
1130
|
-
GetWindowThreadProcessId(target, out targetPid);
|
|
1131
|
-
List<IntPtr> result = new List<IntPtr>();
|
|
1132
|
-
List<IntPtr> all = EnumerateWindows(false);
|
|
1133
|
-
for (int i = 0; i < all.Count; i++)
|
|
1134
|
-
{
|
|
1135
|
-
IntPtr h = all[i];
|
|
1136
|
-
if (h == target) continue;
|
|
1137
|
-
if (!IsWindowVisible(h)) continue;
|
|
1138
|
-
if (IsIconic(h)) continue;
|
|
1139
|
-
uint pid;
|
|
1140
|
-
GetWindowThreadProcessId(h, out pid);
|
|
1141
|
-
if (pid != targetPid) continue;
|
|
1142
|
-
bool owned = GetWindow(h, GW_OWNER) == target;
|
|
1143
|
-
bool popup = (GetWindowLong(h, GWL_STYLE) & WS_POPUP) != 0;
|
|
1144
|
-
if (!owned && !popup) continue;
|
|
1145
|
-
RECT r = BoundsOf(h);
|
|
1146
|
-
bool intersects = r.Left < main.Right && r.Right > main.Left && r.Top < main.Bottom && r.Bottom > main.Top;
|
|
1147
|
-
if (!intersects) continue;
|
|
1148
|
-
result.Add(h);
|
|
1149
|
-
}
|
|
1150
|
-
return result;
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
List<object> CaptureScreenshots(IntPtr h, out bool anyBlank)
|
|
1154
|
-
{
|
|
1155
|
-
anyBlank = false;
|
|
1156
|
-
if (IsIconic(h))
|
|
1157
|
-
throw new ToolError("window_minimized", "window " + h.ToInt64() + " is minimized, so there is nothing to observe: call activate_window (or restore it) and observe again. A minimized window must never be captured, because the image would not be the window.");
|
|
1158
|
-
if (!IsWindowVisible(h))
|
|
1159
|
-
throw new ToolError("window_not_visible", "window " + h.ToInt64() + " is not visible; activate it before observing.");
|
|
1160
|
-
|
|
1161
|
-
List<IntPtr> order = EnumerateWindows(true); // z-order, topmost first
|
|
1162
|
-
List<object> shots = new List<object>();
|
|
1163
|
-
List<IntPtr> targets = new List<IntPtr>();
|
|
1164
|
-
targets.Add(h);
|
|
1165
|
-
targets.AddRange(RelatedWindows(h));
|
|
1166
|
-
|
|
1167
|
-
int total = Math.Max(1, order.Count);
|
|
1168
|
-
for (int i = 0; i < targets.Count; i++)
|
|
1169
|
-
{
|
|
1170
|
-
IntPtr w = targets[i];
|
|
1171
|
-
int zIndex = order.IndexOf(w);
|
|
1172
|
-
if (zIndex < 0) zIndex = total - 1;
|
|
1173
|
-
Dictionary<string, object> shot = CaptureWindow(w);
|
|
1174
|
-
shot["zIndex"] = (long)(total - zIndex); // larger = visually above
|
|
1175
|
-
string id = "shot-" + _shotGeneration + "-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
|
|
1176
|
-
shot["id"] = id;
|
|
1177
|
-
RECT r = BoundsOf(w);
|
|
1178
|
-
lock (_sync)
|
|
1179
|
-
{
|
|
1180
|
-
PruneShots();
|
|
1181
|
-
_shots[id] = new ShotRegion
|
|
1182
|
-
{
|
|
1183
|
-
WindowId = h.ToInt64(), OriginX = r.Left, OriginY = r.Top,
|
|
1184
|
-
Width = (int)ToLong(shot["width"]), Height = (int)ToLong(shot["height"]),
|
|
1185
|
-
ScaleX = ToDouble(shot["scaleX"]), ScaleY = ToDouble(shot["scaleY"]),
|
|
1186
|
-
At = DateTime.UtcNow,
|
|
1187
|
-
};
|
|
1188
|
-
}
|
|
1189
|
-
shots.Add(shot);
|
|
1190
|
-
}
|
|
1191
|
-
return shots;
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
|
-
void PruneShots()
|
|
1195
|
-
{
|
|
1196
|
-
if (_shots.Count <= 48) return;
|
|
1197
|
-
List<string> stale = new List<string>();
|
|
1198
|
-
DateTime cutoff = DateTime.UtcNow.AddMilliseconds(-_shotTtlMs);
|
|
1199
|
-
foreach (KeyValuePair<string, ShotRegion> kv in _shots)
|
|
1200
|
-
if (kv.Value.At < cutoff) stale.Add(kv.Key);
|
|
1201
|
-
for (int i = 0; i < stale.Count; i++) _shots.Remove(stale[i]);
|
|
1202
|
-
}
|
|
1203
|
-
|
|
1204
|
-
ShotRegion ResolveShot(IntPtr h, string screenshotId)
|
|
1205
|
-
{
|
|
1206
|
-
if (string.IsNullOrEmpty(screenshotId))
|
|
1207
|
-
throw new ToolError("bad_request", "screenshotId is empty");
|
|
1208
|
-
ShotRegion region;
|
|
1209
|
-
lock (_sync)
|
|
1210
|
-
{
|
|
1211
|
-
if (!_shots.TryGetValue(screenshotId, out region))
|
|
1212
|
-
throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is no longer cached: it must come from the latest get_window_state for this window. Take a fresh screenshot before retrying.");
|
|
1213
|
-
}
|
|
1214
|
-
if (region.WindowId != h.ToInt64())
|
|
1215
|
-
throw new ToolError("screenshot_mismatch", "screenshot '" + screenshotId + "' belongs to another window; take a fresh observation for this window.");
|
|
1216
|
-
if ((DateTime.UtcNow - region.At).TotalMilliseconds > _shotTtlMs)
|
|
1217
|
-
throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is older than the screenshot TTL. Take a fresh screenshot before retrying.");
|
|
1218
|
-
return region;
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
/// Codex rejects a point that falls outside the screenshot the caller referenced
|
|
1222
|
-
/// ("is outside screenshot bounds"), because that usually means a stale observation.
|
|
1223
|
-
static void AssertPointInsideShot(double x, double y, ShotRegion region, string screenshotId, string what)
|
|
1224
|
-
{
|
|
1225
|
-
if (double.IsNaN(x) || double.IsNaN(y)) return;
|
|
1226
|
-
if (x >= 0 && y >= 0 && x <= region.Width && y <= region.Height) return;
|
|
1227
|
-
throw new ToolError("point_outside_screenshot",
|
|
1228
|
-
what + " (" + x.ToString("0.##") + ", " + y.ToString("0.##") + ") is outside screenshot bounds (0,0)-(" +
|
|
1229
|
-
region.Width + "," + region.Height + ") for '" + screenshotId + "'. Take a fresh screenshot with get_window_state before retrying.");
|
|
1230
|
-
}
|
|
1231
|
-
|
|
1232
|
-
/// Windows.Graphics.Capture through the .NET 8 helper: composed window content,
|
|
1233
|
-
/// occlusion-proof, no DPI halo. Falls back to PrintWindow/screen when absent.
|
|
1234
|
-
static Bitmap TryCaptureWithWgc(IntPtr hwnd)
|
|
1235
|
-
{
|
|
1236
|
-
try
|
|
1237
|
-
{
|
|
1238
|
-
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
|
|
1239
|
-
string exe = Path.Combine(dir, "capture", "dsh-cu-capture.exe");
|
|
1240
|
-
if (!File.Exists(exe)) return null;
|
|
1241
|
-
string tmp = Path.Combine(Path.GetTempPath(), "dsh-cu-wgc-" + Guid.NewGuid().ToString("N") + ".png");
|
|
1242
|
-
try
|
|
1243
|
-
{
|
|
1244
|
-
long budget = Budget.RemainingMs();
|
|
1245
|
-
int waitMs = (int)Math.Max(150, Math.Min(1200, budget - 150));
|
|
1246
|
-
ProcessStartInfo psi = new ProcessStartInfo(exe, "--hwnd " + hwnd.ToInt64().ToString() + " --out \"" + tmp + "\" --timeout " + waitMs);
|
|
1247
|
-
psi.UseShellExecute = false;
|
|
1248
|
-
psi.CreateNoWindow = true;
|
|
1249
|
-
psi.RedirectStandardOutput = true;
|
|
1250
|
-
psi.RedirectStandardError = true;
|
|
1251
|
-
using (Process p = Process.Start(psi))
|
|
1252
|
-
{
|
|
1253
|
-
if (p == null) return null;
|
|
1254
|
-
if (!p.WaitForExit(waitMs + 900)) { try { p.Kill(); } catch { } return null; }
|
|
1255
|
-
if (p.ExitCode != 0 || !File.Exists(tmp)) return null;
|
|
1256
|
-
byte[] bytes = File.ReadAllBytes(tmp);
|
|
1257
|
-
using (MemoryStream ms = new MemoryStream(bytes)) return new Bitmap(ms);
|
|
1258
|
-
}
|
|
1259
|
-
}
|
|
1260
|
-
finally { try { if (File.Exists(tmp)) File.Delete(tmp); } catch { } }
|
|
1261
|
-
}
|
|
1262
|
-
catch { return null; }
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
Dictionary<string, object> CaptureWindow(IntPtr h)
|
|
1266
|
-
{
|
|
1267
|
-
RECT r = BoundsOf(h);
|
|
1268
|
-
int w = Math.Max(1, r.Right - r.Left);
|
|
1269
|
-
int ht = Math.Max(1, r.Bottom - r.Top);
|
|
1270
|
-
// Cap pathological sizes so a maximised multi-monitor window cannot blow the
|
|
1271
|
-
// attachment byte budget.
|
|
1272
|
-
const int cap = 2560;
|
|
1273
|
-
double scale = 1.0;
|
|
1274
|
-
if (w > cap || ht > cap)
|
|
1275
|
-
{
|
|
1276
|
-
scale = Math.Min((double)cap / w, (double)cap / ht);
|
|
1277
|
-
w = Math.Max(1, (int)(w * scale));
|
|
1278
|
-
ht = Math.Max(1, (int)(ht * scale));
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
Bitmap wgc = TryCaptureWithWgc(h);
|
|
1282
|
-
if (wgc != null)
|
|
1283
|
-
{
|
|
1284
|
-
Dictionary<string, object> wg = FinishShot(h, wgc, r, true, 1.0, 1.0);
|
|
1285
|
-
return wg;
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
|
|
1289
|
-
bool printed = false;
|
|
1290
|
-
using (Graphics g = Graphics.FromImage(bmp))
|
|
1291
|
-
{
|
|
1292
|
-
IntPtr hdc = g.GetHdc();
|
|
1293
|
-
try { printed = PrintWindow(h, hdc, 2); }
|
|
1294
|
-
finally { g.ReleaseHdc(hdc); }
|
|
1295
|
-
}
|
|
1296
|
-
if (!printed || IsBlank(bmp))
|
|
1297
|
-
{
|
|
1298
|
-
bmp.Dispose();
|
|
1299
|
-
bmp = CaptureFromScreen(r, w, ht);
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
// DPI-virtualized targets render at their own (usually 96 DPI) size while
|
|
1303
|
-
// occupying a larger physical rect, so PrintWindow leaves a black halo.
|
|
1304
|
-
// Crop to the painted region and remember the physical/screenshot ratio so
|
|
1305
|
-
// coordinate actions can map screenshot pixels back to screen pixels.
|
|
1306
|
-
int fullWidth = bmp.Width, fullHeight = bmp.Height;
|
|
1307
|
-
Rectangle painted = PaintedBounds(bmp);
|
|
1308
|
-
if (painted.Width > 16 && painted.Height > 16 && (painted.Width < bmp.Width || painted.Height < bmp.Height))
|
|
1309
|
-
{
|
|
1310
|
-
Bitmap cropped = bmp.Clone(new Rectangle(0, 0, painted.Right, painted.Bottom), bmp.PixelFormat);
|
|
1311
|
-
bmp.Dispose();
|
|
1312
|
-
bmp = cropped;
|
|
1313
|
-
w = bmp.Width;
|
|
1314
|
-
ht = bmp.Height;
|
|
1315
|
-
}
|
|
1316
|
-
double scaleX = (double)fullWidth / Math.Max(1, bmp.Width);
|
|
1317
|
-
double scaleY = (double)fullHeight / Math.Max(1, bmp.Height);
|
|
1318
|
-
|
|
1319
|
-
byte[] png;
|
|
1320
|
-
using (MemoryStream ms = new MemoryStream())
|
|
1321
|
-
{
|
|
1322
|
-
bmp.Save(ms, ImageFormat.Png);
|
|
1323
|
-
png = ms.ToArray();
|
|
1324
|
-
}
|
|
1325
|
-
bmp.Dispose();
|
|
1326
|
-
|
|
1327
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1328
|
-
d["mediaType"] = "image/png";
|
|
1329
|
-
d["base64"] = Convert.ToBase64String(png);
|
|
1330
|
-
d["width"] = (long)w;
|
|
1331
|
-
d["height"] = (long)ht;
|
|
1332
|
-
d["originX"] = (long)r.Left;
|
|
1333
|
-
d["originY"] = (long)r.Top;
|
|
1334
|
-
d["zIndex"] = 1L;
|
|
1335
|
-
d["printed"] = printed;
|
|
1336
|
-
d["scaleX"] = scaleX;
|
|
1337
|
-
d["scaleY"] = scaleY;
|
|
1338
|
-
return d;
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
|
-
static Dictionary<string, object> FinishShot(IntPtr h, Bitmap bmp, RECT r, bool printed, double scaleX, double scaleY)
|
|
1342
|
-
{
|
|
1343
|
-
int w = bmp.Width, ht = bmp.Height;
|
|
1344
|
-
byte[] png;
|
|
1345
|
-
using (MemoryStream ms = new MemoryStream())
|
|
1346
|
-
{
|
|
1347
|
-
bmp.Save(ms, ImageFormat.Png);
|
|
1348
|
-
png = ms.ToArray();
|
|
1349
|
-
}
|
|
1350
|
-
bmp.Dispose();
|
|
1351
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1352
|
-
d["mediaType"] = "image/png";
|
|
1353
|
-
d["base64"] = Convert.ToBase64String(png);
|
|
1354
|
-
d["width"] = (long)w;
|
|
1355
|
-
d["height"] = (long)ht;
|
|
1356
|
-
d["originX"] = (long)r.Left;
|
|
1357
|
-
d["originY"] = (long)r.Top;
|
|
1358
|
-
d["zIndex"] = 1L;
|
|
1359
|
-
d["printed"] = printed;
|
|
1360
|
-
d["scaleX"] = scaleX;
|
|
1361
|
-
d["scaleY"] = scaleY;
|
|
1362
|
-
return d;
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
static Bitmap CaptureFromScreen(RECT r, int w, int ht)
|
|
1366
|
-
{
|
|
1367
|
-
Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
|
|
1368
|
-
using (Graphics g = Graphics.FromImage(bmp))
|
|
1369
|
-
{
|
|
1370
|
-
try { g.CopyFromScreen(r.Left, r.Top, 0, 0, new Size(w, ht), CopyPixelOperation.SourceCopy); }
|
|
1371
|
-
catch { }
|
|
1372
|
-
}
|
|
1373
|
-
return bmp;
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
/// Bounding box of everything that is not uniform black, i.e. the region the
|
|
1377
|
-
/// target actually painted. Used to strip the DPI-virtualization halo.
|
|
1378
|
-
static Rectangle PaintedBounds(Bitmap bmp)
|
|
1379
|
-
{
|
|
1380
|
-
int right = 0, bottom = 0;
|
|
1381
|
-
int step = Math.Max(1, Math.Min(bmp.Width, bmp.Height) / 200);
|
|
1382
|
-
for (int y = 0; y < bmp.Height; y += step)
|
|
1383
|
-
{
|
|
1384
|
-
for (int x = 0; x < bmp.Width; x += step)
|
|
1385
|
-
{
|
|
1386
|
-
Color c = bmp.GetPixel(x, y);
|
|
1387
|
-
if (c.R > 8 || c.G > 8 || c.B > 8)
|
|
1388
|
-
{
|
|
1389
|
-
if (x > right) right = x;
|
|
1390
|
-
if (y > bottom) bottom = y;
|
|
1391
|
-
}
|
|
1392
|
-
}
|
|
1393
|
-
}
|
|
1394
|
-
return new Rectangle(0, 0, Math.Min(bmp.Width, right + step), Math.Min(bmp.Height, bottom + step));
|
|
1395
|
-
}
|
|
1396
|
-
|
|
1397
|
-
static bool IsBlank(Bitmap bmp)
|
|
1398
|
-
{
|
|
1399
|
-
int samples = 0, dark = 0;
|
|
1400
|
-
int stepX = Math.Max(1, bmp.Width / 12);
|
|
1401
|
-
int stepY = Math.Max(1, bmp.Height / 12);
|
|
1402
|
-
for (int y = 0; y < bmp.Height; y += stepY)
|
|
1403
|
-
{
|
|
1404
|
-
for (int x = 0; x < bmp.Width; x += stepX)
|
|
1405
|
-
{
|
|
1406
|
-
Color c = bmp.GetPixel(x, y);
|
|
1407
|
-
samples++;
|
|
1408
|
-
if (c.R < 6 && c.G < 6 && c.B < 6) dark++;
|
|
1409
|
-
}
|
|
1410
|
-
}
|
|
1411
|
-
return samples > 0 && dark == samples;
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
// ---- element resolution ----
|
|
1415
|
-
AutomationElement ElementAt(IntPtr hwnd, long index)
|
|
1416
|
-
{
|
|
1417
|
-
Snapshot snap;
|
|
1418
|
-
lock (_sync)
|
|
1419
|
-
{
|
|
1420
|
-
if (!_snapshots.TryGetValue(hwnd.ToInt64(), out snap))
|
|
1421
|
-
throw new ToolError("stale_snapshot", "no observation for this window yet: call get_window_state with include_text=true before using element_index");
|
|
1422
|
-
}
|
|
1423
|
-
if ((DateTime.UtcNow - snap.At).TotalMilliseconds > _snapshotTtlMs)
|
|
1424
|
-
throw new ToolError("stale_snapshot", "the element snapshot is older than " + _snapshotTtlMs + "ms: call get_window_state again before acting on element indexes");
|
|
1425
|
-
AutomationElement el;
|
|
1426
|
-
if (!snap.Elements.TryGetValue(index, out el))
|
|
1427
|
-
throw new ToolError("bad_index", "element index " + index + " is not part of the latest observation for this window");
|
|
1428
|
-
return el;
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
static Dictionary<string, object> ScreenshotTarget(IntPtr h, AutomationElement el)
|
|
1432
|
-
{
|
|
1433
|
-
RECT obs = BoundsOf(h);
|
|
1434
|
-
System.Windows.Rect r = el.Current.BoundingRectangle;
|
|
1435
|
-
if (r.IsEmpty || r.Width <= 0 || r.Height <= 0)
|
|
1436
|
-
throw new ToolError("no_geometry", "the element has no bounding rectangle; use x/y coordinates instead");
|
|
1437
|
-
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1438
|
-
d["screenX"] = (int)r.X + (int)(r.Width / 2);
|
|
1439
|
-
d["screenY"] = (int)r.Y + (int)(r.Height / 2);
|
|
1440
|
-
d["originX"] = obs.Left;
|
|
1441
|
-
d["originY"] = obs.Top;
|
|
1442
|
-
return d;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
// ---- input ----
|
|
1446
|
-
static void SendMouse(uint flags) { SendMouseData(flags, 0); }
|
|
1447
|
-
|
|
1448
|
-
static void SendMouseData(uint flags, int data)
|
|
1449
|
-
{
|
|
1450
|
-
INPUT inp = new INPUT();
|
|
1451
|
-
inp.type = INPUT_MOUSE;
|
|
1452
|
-
inp.u.mi.dx = 0; inp.u.mi.dy = 0;
|
|
1453
|
-
inp.u.mi.mouseData = (uint)data;
|
|
1454
|
-
inp.u.mi.dwFlags = flags;
|
|
1455
|
-
INPUT[] arr = new INPUT[] { inp };
|
|
1456
|
-
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
static void KeyDown(ushort vk) { SendKey(vk, 0, 0); }
|
|
1460
|
-
static void KeyUp(ushort vk) { SendKey(vk, 0, KEYEVENTF_KEYUP); }
|
|
1461
|
-
|
|
1462
|
-
static void SendKey(ushort vk, ushort scan, uint flags)
|
|
1463
|
-
{
|
|
1464
|
-
INPUT inp = new INPUT();
|
|
1465
|
-
inp.type = INPUT_KEYBOARD;
|
|
1466
|
-
inp.u.ki.wVk = vk;
|
|
1467
|
-
inp.u.ki.wScan = scan;
|
|
1468
|
-
inp.u.ki.dwFlags = flags;
|
|
1469
|
-
INPUT[] arr = new INPUT[] { inp };
|
|
1470
|
-
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
static void SendUnicode(char ch, bool up)
|
|
1474
|
-
{
|
|
1475
|
-
INPUT inp = new INPUT();
|
|
1476
|
-
inp.type = INPUT_KEYBOARD;
|
|
1477
|
-
inp.u.ki.wVk = 0;
|
|
1478
|
-
inp.u.ki.wScan = ch;
|
|
1479
|
-
inp.u.ki.dwFlags = KEYEVENTF_UNICODE | (up ? KEYEVENTF_KEYUP : 0);
|
|
1480
|
-
INPUT[] arr = new INPUT[] { inp };
|
|
1481
|
-
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1482
|
-
}
|
|
1483
|
-
|
|
1484
|
-
void ClickAt(int screenX, int screenY, int count, string button, Dictionary<string, object> p)
|
|
1485
|
-
{
|
|
1486
|
-
SetCursorPos(screenX, screenY);
|
|
1487
|
-
System.Threading.Thread.Sleep(35);
|
|
1488
|
-
for (int i = 0; i < Math.Max(1, count); i++)
|
|
1489
|
-
{
|
|
1490
|
-
CheckCancel(p);
|
|
1491
|
-
if (i > 0) System.Threading.Thread.Sleep(60);
|
|
1492
|
-
uint down = MOUSEEVENTF_LEFTDOWN, up = MOUSEEVENTF_LEFTUP;
|
|
1493
|
-
if (button == "r") { down = MOUSEEVENTF_RIGHTDOWN; up = MOUSEEVENTF_RIGHTUP; }
|
|
1494
|
-
else if (button == "m") { down = MOUSEEVENTF_MIDDLEDOWN; up = MOUSEEVENTF_MIDDLEUP; }
|
|
1495
|
-
SendMouse(down);
|
|
1496
|
-
try { System.Threading.Thread.Sleep(22); }
|
|
1497
|
-
finally { SendMouse(up); }
|
|
1498
|
-
}
|
|
1499
|
-
}
|
|
1500
|
-
|
|
1501
|
-
public Dictionary<string, object> Click(Dictionary<string, object> p)
|
|
1502
|
-
{
|
|
1503
|
-
CheckCancel(p);
|
|
1504
|
-
AssertUnlocked();
|
|
1505
|
-
EnsureOverlay(8000);
|
|
1506
|
-
IntPtr h = ResolveWindow(p);
|
|
1507
|
-
AssertAppApproved(h);
|
|
1508
|
-
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1509
|
-
string button = NormalizeButton(Json.GetString(p, "mouse_button"));
|
|
1510
|
-
int count = (int)Json.GetLong(p, "click_count", 1);
|
|
1511
|
-
int sx, sy;
|
|
1512
|
-
|
|
1513
|
-
if (elementIndex >= 0)
|
|
1514
|
-
{
|
|
1515
|
-
AutomationElement el = ElementAt(h, elementIndex);
|
|
1516
|
-
Dictionary<string, object> t = ScreenshotTarget(h, el);
|
|
1517
|
-
sx = (int)t["screenX"]; sy = (int)t["screenY"];
|
|
1518
|
-
// Prefer the semantic invoke when the element exposes one and we are
|
|
1519
|
-
// doing a plain left single click: it survives occlusion and does not
|
|
1520
|
-
// depend on pixel geometry.
|
|
1521
|
-
if (count == 1 && button == "l")
|
|
1522
|
-
{
|
|
1523
|
-
object pattern;
|
|
1524
|
-
try
|
|
1525
|
-
{
|
|
1526
|
-
if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1527
|
-
{
|
|
1528
|
-
((InvokePattern)pattern).Invoke();
|
|
1529
|
-
Dictionary<string, object> r2 = new Dictionary<string, object>();
|
|
1530
|
-
r2["clicked"] = "invoke"; r2["element_index"] = elementIndex;
|
|
1531
|
-
return r2;
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1534
|
-
catch (Exception) { }
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
else
|
|
1538
|
-
{
|
|
1539
|
-
double x = Json.GetDouble(p, "x", double.NaN);
|
|
1540
|
-
double y = Json.GetDouble(p, "y", double.NaN);
|
|
1541
|
-
if (double.IsNaN(x) || double.IsNaN(y)) throw new ToolError("bad_request", "click needs either 'element_index' or both 'x' and 'y' (window-relative pixels)");
|
|
1542
|
-
string screenshotId = Json.GetString(p, "screenshotId");
|
|
1543
|
-
int originX, originY;
|
|
1544
|
-
double coordScaleX = 1.0, coordScaleY = 1.0;
|
|
1545
|
-
if (!string.IsNullOrEmpty(screenshotId))
|
|
1546
|
-
{
|
|
1547
|
-
ShotRegion region = ResolveShot(h, screenshotId);
|
|
1548
|
-
AssertPointInsideShot(x, y, region, screenshotId, "click");
|
|
1549
|
-
originX = region.OriginX; originY = region.OriginY;
|
|
1550
|
-
coordScaleX = region.ScaleX; coordScaleY = region.ScaleY;
|
|
1551
|
-
}
|
|
1552
|
-
else
|
|
1553
|
-
{
|
|
1554
|
-
RECT bounds = BoundsOf(h);
|
|
1555
|
-
originX = bounds.Left; originY = bounds.Top;
|
|
1556
|
-
}
|
|
1557
|
-
sx = originX + (int)Math.Round(x * coordScaleX);
|
|
1558
|
-
sy = originY + (int)Math.Round(y * coordScaleY);
|
|
1559
|
-
}
|
|
1560
|
-
|
|
1561
|
-
Activate(h); // pointer clicks need a real foreground window
|
|
1562
|
-
MarkCursor(sx, sy);
|
|
1563
|
-
CheckCancel(p);
|
|
1564
|
-
ClickAt(sx, sy, count, button, p);
|
|
1565
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1566
|
-
r["clicked"] = "pointer";
|
|
1567
|
-
r["screenX"] = (long)sx;
|
|
1568
|
-
r["screenY"] = (long)sy;
|
|
1569
|
-
r["clickCount"] = (long)count;
|
|
1570
|
-
return r;
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
static string NormalizeButton(string b)
|
|
1574
|
-
{
|
|
1575
|
-
if (string.IsNullOrEmpty(b)) return "l";
|
|
1576
|
-
string s = b.Trim().ToLowerInvariant();
|
|
1577
|
-
if (s == "right" || s == "r") return "r";
|
|
1578
|
-
if (s == "middle" || s == "m") return "m";
|
|
1579
|
-
return "l";
|
|
1580
|
-
}
|
|
1581
|
-
|
|
1582
|
-
public Dictionary<string, object> PressKey(Dictionary<string, object> p)
|
|
1583
|
-
{
|
|
1584
|
-
CheckCancel(p);
|
|
1585
|
-
AssertUnlocked();
|
|
1586
|
-
EnsureOverlay(8000);
|
|
1587
|
-
IntPtr h = ResolveWindow(p);
|
|
1588
|
-
AssertAppApproved(h);
|
|
1589
|
-
string key = Json.GetString(p, "key");
|
|
1590
|
-
if (string.IsNullOrEmpty(key)) throw new ToolError("bad_request", "press_key requires 'key'");
|
|
1591
|
-
Activate(h);
|
|
1592
|
-
Chord(key, p);
|
|
1593
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1594
|
-
r["pressed"] = key;
|
|
1595
|
-
return r;
|
|
1596
|
-
}
|
|
1597
|
-
|
|
1598
|
-
static readonly Dictionary<string, ushort> KeyMap = BuildKeyMap();
|
|
1599
|
-
|
|
1600
|
-
static Dictionary<string, ushort> BuildKeyMap()
|
|
1601
|
-
{
|
|
1602
|
-
Dictionary<string, ushort> m = new Dictionary<string, ushort>(StringComparer.OrdinalIgnoreCase);
|
|
1603
|
-
for (char c = 'a'; c <= 'z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'a'));
|
|
1604
|
-
for (char c = 'A'; c <= 'Z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'A'));
|
|
1605
|
-
for (char c = '0'; c <= '9'; c++) m[c.ToString()] = (ushort)(0x30 + (c - '0'));
|
|
1606
|
-
m["return"] = 0x0D; m["enter"] = 0x0D; m["kp_enter"] = 0x0D;
|
|
1607
|
-
m["tab"] = 0x09; m["space"] = 0x20; m["backspace"] = 0x08; m["escape"] = 0x1B; m["esc"] = 0x1B;
|
|
1608
|
-
m["delete"] = 0x2E; m["insert"] = 0x2D; m["home"] = 0x24; m["end"] = 0x23;
|
|
1609
|
-
m["page_up"] = 0x21; m["pageup"] = 0x21; m["prior"] = 0x21; m["page_down"] = 0x22; m["pagedown"] = 0x22; m["next"] = 0x22;
|
|
1610
|
-
m["left"] = 0x25; m["up"] = 0x26; m["right"] = 0x27; m["down"] = 0x28;
|
|
1611
|
-
m["shift_l"] = 0xA0; m["shift"] = 0xA0; m["shift_r"] = 0xA1;
|
|
1612
|
-
m["control_l"] = 0xA2; m["control"] = 0xA2; m["ctrl"] = 0xA2; m["control_r"] = 0xA3;
|
|
1613
|
-
m["alt_l"] = 0xA4; m["alt"] = 0xA4; m["alt_r"] = 0xA5;
|
|
1614
|
-
m["super_l"] = 0x5B; m["win"] = 0x5B; m["meta"] = 0x5B; m["lwin"] = 0x5B; m["super"] = 0x5B;
|
|
1615
|
-
m["super_r"] = 0x5C; m["rwin"] = 0x5C;
|
|
1616
|
-
m["caps_lock"] = 0x14; m["num_lock"] = 0x90; m["scroll_lock"] = 0x91;
|
|
1617
|
-
m["menu"] = 0x5D; m["apps"] = 0x5D; m["print"] = 0x2C; m["pause"] = 0x13;
|
|
1618
|
-
m["period"] = 0xBE; m["greater"] = 0xBE; m["comma"] = 0xBC; m["less"] = 0xBC;
|
|
1619
|
-
m["slash"] = 0xBF; m["question"] = 0xBF; m["backslash"] = 0xDC; m["bar"] = 0xDC;
|
|
1620
|
-
m["semicolon"] = 0xBA; m["colon"] = 0xBA; m["apostrophe"] = 0xDE; m["quote"] = 0xDE;
|
|
1621
|
-
m["minus"] = 0xBD; m["underscore"] = 0xBD; m["equal"] = 0xBB; m["plus"] = 0xBB;
|
|
1622
|
-
m["bracketleft"] = 0xDB; m["bracketright"] = 0xDD; m["grave"] = 0xC0; m["asciitilde"] = 0xC0;
|
|
1623
|
-
for (int i = 1; i <= 24; i++) { m["f" + i] = (ushort)(0x6F + i); }
|
|
1624
|
-
// Numpad: both X keysym style (KP_0) and the aliases the API documents.
|
|
1625
|
-
for (int i = 0; i <= 9; i++)
|
|
1626
|
-
{
|
|
1627
|
-
m["kp_" + i] = (ushort)(0x60 + i);
|
|
1628
|
-
m["numpad_" + i] = (ushort)(0x60 + i);
|
|
1629
|
-
}
|
|
1630
|
-
m["kp_add"] = 0x6B; m["numpad_add"] = 0x6B; m["plus_numpad"] = 0x6B;
|
|
1631
|
-
m["kp_subtract"] = 0x6D; m["numpad_subtract"] = 0x6D; m["minus_numpad"] = 0x6D;
|
|
1632
|
-
m["kp_multiply"] = 0x6A; m["numpad_multiply"] = 0x6A; m["kp_asterisk"] = 0x6A;
|
|
1633
|
-
m["kp_divide"] = 0x6F; m["numpad_divide"] = 0x6F; m["kp_slash"] = 0x6F;
|
|
1634
|
-
m["kp_decimal"] = 0x6E; m["numpad_decimal"] = 0x6E; m["kp_separator"] = 0x6C;
|
|
1635
|
-
m["numpad_enter"] = 0x0D;
|
|
1636
|
-
return m;
|
|
1637
|
-
}
|
|
1638
|
-
|
|
1639
|
-
static readonly HashSet<ushort> ExtendedKeys = new HashSet<ushort>(new ushort[] {
|
|
1640
|
-
0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x2D,0x2E,0x5B,0x5C,0x5D,0x90,0x6F,0xA3,0xA5
|
|
1641
|
-
});
|
|
1642
|
-
|
|
1643
|
-
static void SendVk(ushort vk, bool up)
|
|
1644
|
-
{
|
|
1645
|
-
uint flags = up ? KEYEVENTF_KEYUP : 0;
|
|
1646
|
-
if (ExtendedKeys.Contains(vk)) flags |= KEYEVENTF_EXTENDEDKEY;
|
|
1647
|
-
SendKey(vk, 0, flags);
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
void Chord(string chord, Dictionary<string, object> p)
|
|
1651
|
-
{
|
|
1652
|
-
string[] parts = chord.Split('+');
|
|
1653
|
-
List<ushort> keys = new List<ushort>();
|
|
1654
|
-
foreach (string raw in parts)
|
|
1655
|
-
{
|
|
1656
|
-
string token = raw.Trim();
|
|
1657
|
-
if (token.Length == 0) continue;
|
|
1658
|
-
keys.Add(ResolveKey(token));
|
|
1659
|
-
}
|
|
1660
|
-
if (keys.Count == 0) throw new ToolError("bad_request", "empty key chord");
|
|
1661
|
-
int pressed = 0;
|
|
1662
|
-
try
|
|
1663
|
-
{
|
|
1664
|
-
for (int i = 0; i < keys.Count; i++)
|
|
1665
|
-
{
|
|
1666
|
-
CheckCancel(p);
|
|
1667
|
-
SendVk(keys[i], false); pressed++;
|
|
1668
|
-
System.Threading.Thread.Sleep(12);
|
|
1669
|
-
}
|
|
1670
|
-
}
|
|
1671
|
-
finally { for (int i = pressed - 1; i >= 0; i--) SendVk(keys[i], true); }
|
|
1672
|
-
}
|
|
1673
|
-
|
|
1674
|
-
/// The Windows/Meta key family is hard-denied: it opens Run, Settings and
|
|
1675
|
-
/// shell surfaces that Computer Use must never drive. Enforced in the helper
|
|
1676
|
-
/// as well as in the plugin, so a direct protocol call cannot bypass it.
|
|
1677
|
-
static readonly HashSet<string> DeniedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
|
|
1678
|
-
"win", "win_l", "win_r", "lwin", "rwin", "windows", "windows_l", "windows_r",
|
|
1679
|
-
"super", "super_l", "super_r", "meta", "meta_l", "meta_r", "cmd", "command", "os", "os_l", "os_r"
|
|
1680
|
-
};
|
|
1681
|
-
|
|
1682
|
-
static ushort ResolveKey(string token)
|
|
1683
|
-
{
|
|
1684
|
-
if (DeniedKeys.Contains(token))
|
|
1685
|
-
throw new ToolError("hard_denied", "the Windows/Meta/Super key ('" + token + "') is hard-denied: it opens Run, Settings and shell surfaces that Computer Use must never drive.");
|
|
1686
|
-
ushort vk;
|
|
1687
|
-
if (KeyMap.TryGetValue(token, out vk)) return vk;
|
|
1688
|
-
if (token.Length == 1)
|
|
1689
|
-
{
|
|
1690
|
-
char c = token[0];
|
|
1691
|
-
short scan = VkKeyScan(c);
|
|
1692
|
-
if (scan != -1) return (ushort)(scan & 0xFF);
|
|
1693
|
-
return (ushort)char.ToUpperInvariant(c);
|
|
1694
|
-
}
|
|
1695
|
-
throw new ToolError("bad_key", "unknown key name '" + token + "'");
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
[DllImport("user32.dll")] static extern short VkKeyScan(char ch);
|
|
1699
|
-
|
|
1700
|
-
public Dictionary<string, object> TypeText(Dictionary<string, object> p)
|
|
1701
|
-
{
|
|
1702
|
-
CheckCancel(p);
|
|
1703
|
-
AssertUnlocked();
|
|
1704
|
-
EnsureOverlay(8000);
|
|
1705
|
-
IntPtr h = ResolveWindow(p);
|
|
1706
|
-
AssertAppApproved(h);
|
|
1707
|
-
string text = Json.GetString(p, "text");
|
|
1708
|
-
if (text == null) throw new ToolError("bad_request", "type_text requires 'text'");
|
|
1709
|
-
Activate(h);
|
|
1710
|
-
for (int i = 0; i < text.Length; i++)
|
|
1711
|
-
{
|
|
1712
|
-
CheckCancel(p);
|
|
1713
|
-
char c = text[i];
|
|
1714
|
-
if (c == '\n') { SendVk(0x0D, false); SendVk(0x0D, true); continue; }
|
|
1715
|
-
if (c == '\t') { SendVk(0x09, false); SendVk(0x09, true); continue; }
|
|
1716
|
-
if (c == '\r') continue;
|
|
1717
|
-
SendUnicode(c, false);
|
|
1718
|
-
SendUnicode(c, true);
|
|
1719
|
-
if ((i & 15) == 0) System.Threading.Thread.Sleep(4);
|
|
1720
|
-
}
|
|
1721
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1722
|
-
r["typed"] = (long)text.Length;
|
|
1723
|
-
return r;
|
|
1724
|
-
}
|
|
1725
|
-
|
|
1726
|
-
public Dictionary<string, object> Scroll(Dictionary<string, object> p)
|
|
1727
|
-
{
|
|
1728
|
-
CheckCancel(p);
|
|
1729
|
-
AssertUnlocked();
|
|
1730
|
-
EnsureOverlay(8000);
|
|
1731
|
-
IntPtr h = ResolveWindow(p);
|
|
1732
|
-
AssertAppApproved(h);
|
|
1733
|
-
// Validation first: an invalid call must not activate a window or move the pointer.
|
|
1734
|
-
double scrollY = Json.GetDouble(p, "scrollY", 0);
|
|
1735
|
-
double scrollX = Json.GetDouble(p, "scrollX", 0);
|
|
1736
|
-
double x = Json.GetDouble(p, "x", double.NaN);
|
|
1737
|
-
double y = Json.GetDouble(p, "y", double.NaN);
|
|
1738
|
-
if (double.IsNaN(x) || double.IsNaN(y))
|
|
1739
|
-
throw new ToolError("bad_request", "scroll requires finite 'x' and 'y' (the point to scroll from, in window/screenshot coordinates); click a pane first if it needs focus");
|
|
1740
|
-
if (Math.Abs(scrollX) < 0.0001 && Math.Abs(scrollY) < 0.0001)
|
|
1741
|
-
throw new ToolError("bad_request", "scroll requires a non-zero scrollX or scrollY");
|
|
1742
|
-
Activate(h);
|
|
1743
|
-
int sx, sy;
|
|
1744
|
-
string scrollShot = Json.GetString(p, "screenshotId");
|
|
1745
|
-
int originLeft, originTop;
|
|
1746
|
-
double scrollScaleX = 1.0, scrollScaleY = 1.0;
|
|
1747
|
-
if (!string.IsNullOrEmpty(scrollShot))
|
|
1748
|
-
{
|
|
1749
|
-
ShotRegion region = ResolveShot(h, scrollShot);
|
|
1750
|
-
AssertPointInsideShot(x, y, region, scrollShot, "scroll origin");
|
|
1751
|
-
originLeft = region.OriginX; originTop = region.OriginY;
|
|
1752
|
-
scrollScaleX = region.ScaleX; scrollScaleY = region.ScaleY;
|
|
1753
|
-
}
|
|
1754
|
-
else
|
|
1755
|
-
{
|
|
1756
|
-
RECT b = BoundsOf(h);
|
|
1757
|
-
originLeft = b.Left; originTop = b.Top;
|
|
1758
|
-
}
|
|
1759
|
-
sx = originLeft + (int)Math.Round(x * scrollScaleX);
|
|
1760
|
-
sy = originTop + (int)Math.Round(y * scrollScaleY);
|
|
1761
|
-
MarkCursor(sx, sy);
|
|
1762
|
-
SetCursorPos(sx, sy);
|
|
1763
|
-
System.Threading.Thread.Sleep(40);
|
|
1764
|
-
if (Math.Abs(scrollY) > 0.0001) SendMouseData(MOUSEEVENTF_WHEEL, (int)(-scrollY * 120));
|
|
1765
|
-
if (Math.Abs(scrollX) > 0.0001) SendMouseData(MOUSEEVENTF_HWHEEL, (int)(-scrollX * 120));
|
|
1766
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1767
|
-
r["scrolled"] = true;
|
|
1768
|
-
r["screenX"] = (long)sx; r["screenY"] = (long)sy;
|
|
1769
|
-
return r;
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
|
-
public Dictionary<string, object> Drag(Dictionary<string, object> p)
|
|
1773
|
-
{
|
|
1774
|
-
CheckCancel(p);
|
|
1775
|
-
AssertUnlocked();
|
|
1776
|
-
EnsureOverlay(8000);
|
|
1777
|
-
IntPtr h = ResolveWindow(p);
|
|
1778
|
-
AssertAppApproved(h);
|
|
1779
|
-
double fromX = Json.GetDouble(p, "from_x", double.NaN);
|
|
1780
|
-
double fromY = Json.GetDouble(p, "from_y", double.NaN);
|
|
1781
|
-
double toX = Json.GetDouble(p, "to_x", double.NaN);
|
|
1782
|
-
double toY = Json.GetDouble(p, "to_y", double.NaN);
|
|
1783
|
-
if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
|
|
1784
|
-
throw new ToolError("bad_request", "drag requires finite from_x/from_y/to_x/to_y (window-relative pixels)");
|
|
1785
|
-
Activate(h);
|
|
1786
|
-
int originX, originY;
|
|
1787
|
-
double dragScaleX = 1.0, dragScaleY = 1.0;
|
|
1788
|
-
string dragShot = Json.GetString(p, "screenshotId");
|
|
1789
|
-
if (!string.IsNullOrEmpty(dragShot))
|
|
1790
|
-
{
|
|
1791
|
-
ShotRegion region = ResolveShot(h, dragShot);
|
|
1792
|
-
AssertPointInsideShot(fromX, fromY, region, dragShot, "drag start");
|
|
1793
|
-
AssertPointInsideShot(toX, toY, region, dragShot, "drag end");
|
|
1794
|
-
originX = region.OriginX; originY = region.OriginY;
|
|
1795
|
-
dragScaleX = region.ScaleX; dragScaleY = region.ScaleY;
|
|
1796
|
-
}
|
|
1797
|
-
else
|
|
1798
|
-
{
|
|
1799
|
-
RECT b = BoundsOf(h);
|
|
1800
|
-
originX = b.Left; originY = b.Top;
|
|
1801
|
-
}
|
|
1802
|
-
int fx = originX + (int)Math.Round(fromX * dragScaleX);
|
|
1803
|
-
int fy = originY + (int)Math.Round(fromY * dragScaleY);
|
|
1804
|
-
int tx = originX + (int)Math.Round(toX * dragScaleX);
|
|
1805
|
-
int ty = originY + (int)Math.Round(toY * dragScaleY);
|
|
1806
|
-
|
|
1807
|
-
MarkCursor(fx, fy);
|
|
1808
|
-
SetCursorPos(fx, fy);
|
|
1809
|
-
System.Threading.Thread.Sleep(60);
|
|
1810
|
-
CheckCancel(p);
|
|
1811
|
-
SendMouse(MOUSEEVENTF_LEFTDOWN);
|
|
1812
|
-
try
|
|
1813
|
-
{
|
|
1814
|
-
System.Threading.Thread.Sleep(80);
|
|
1815
|
-
const int steps = 14;
|
|
1816
|
-
for (int i = 1; i <= steps; i++)
|
|
1817
|
-
{
|
|
1818
|
-
CheckCancel(p);
|
|
1819
|
-
int x = fx + (tx - fx) * i / steps;
|
|
1820
|
-
int y = fy + (ty - fy) * i / steps;
|
|
1821
|
-
SetCursorPos(x, y);
|
|
1822
|
-
System.Threading.Thread.Sleep(18);
|
|
1823
|
-
}
|
|
1824
|
-
System.Threading.Thread.Sleep(90);
|
|
1825
|
-
CheckCancel(p);
|
|
1826
|
-
}
|
|
1827
|
-
finally { SendMouse(MOUSEEVENTF_LEFTUP); }
|
|
1828
|
-
|
|
1829
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1830
|
-
r["dragged"] = true;
|
|
1831
|
-
r["from"] = new long[] { fx, fy };
|
|
1832
|
-
r["to"] = new long[] { tx, ty };
|
|
1833
|
-
return r;
|
|
1834
|
-
}
|
|
1835
|
-
|
|
1836
|
-
public Dictionary<string, object> SetValue(Dictionary<string, object> p)
|
|
1837
|
-
{
|
|
1838
|
-
CheckCancel(p);
|
|
1839
|
-
AssertUnlocked();
|
|
1840
|
-
EnsureOverlay(8000);
|
|
1841
|
-
IntPtr h = ResolveWindow(p);
|
|
1842
|
-
AssertAppApproved(h);
|
|
1843
|
-
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1844
|
-
if (elementIndex < 0) throw new ToolError("bad_request", "set_value requires 'element_index'");
|
|
1845
|
-
string value = Json.GetString(p, "value");
|
|
1846
|
-
if (value == null) throw new ToolError("bad_request", "set_value requires 'value'");
|
|
1847
|
-
AutomationElement el = ElementAt(h, elementIndex);
|
|
1848
|
-
object pattern;
|
|
1849
|
-
if (!el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern))
|
|
1850
|
-
throw new ToolError("pattern_unavailable", "element [" + elementIndex + "] does not expose the Value pattern; use click + type_text instead");
|
|
1851
|
-
((ValuePattern)pattern).SetValue(value);
|
|
1852
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1853
|
-
r["valueSet"] = true;
|
|
1854
|
-
r["element_index"] = elementIndex;
|
|
1855
|
-
return r;
|
|
1856
|
-
}
|
|
1857
|
-
|
|
1858
|
-
public Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p)
|
|
1859
|
-
{
|
|
1860
|
-
CheckCancel(p);
|
|
1861
|
-
AssertUnlocked();
|
|
1862
|
-
EnsureOverlay(8000);
|
|
1863
|
-
IntPtr h = ResolveWindow(p);
|
|
1864
|
-
AssertAppApproved(h);
|
|
1865
|
-
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1866
|
-
string action = Json.GetString(p, "action");
|
|
1867
|
-
if (elementIndex < 0 || string.IsNullOrEmpty(action))
|
|
1868
|
-
throw new ToolError("bad_request", "perform_secondary_action requires 'element_index' and 'action'");
|
|
1869
|
-
AutomationElement el = ElementAt(h, elementIndex);
|
|
1870
|
-
string a = action.Trim().ToLowerInvariant();
|
|
1871
|
-
object pattern;
|
|
1872
|
-
|
|
1873
|
-
if ((a == "raise" || a == "focus") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1874
|
-
{ ((InvokePattern)pattern).Invoke(); }
|
|
1875
|
-
else if (a == "raise" || a == "focus") { try { el.SetFocus(); } catch (Exception ex) { throw new ToolError("action_failed", ex.Message); } }
|
|
1876
|
-
else if ((a == "invoke" || a == "press") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1877
|
-
{ ((InvokePattern)pattern).Invoke(); }
|
|
1878
|
-
else if ((a == "expand" || a == "collapse") && el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern))
|
|
1879
|
-
{
|
|
1880
|
-
ExpandCollapsePattern ec = (ExpandCollapsePattern)pattern;
|
|
1881
|
-
if (a == "expand") ec.Expand(); else ec.Collapse();
|
|
1882
|
-
}
|
|
1883
|
-
else if ((a == "toggle") && el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern))
|
|
1884
|
-
{ ((TogglePattern)pattern).Toggle(); }
|
|
1885
|
-
else if ((a == "select" || a == "selectitem") && el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern))
|
|
1886
|
-
{ ((SelectionItemPattern)pattern).Select(); }
|
|
1887
|
-
else if (a.StartsWith("scroll") && el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern))
|
|
1888
|
-
{
|
|
1889
|
-
ScrollPattern sp = (ScrollPattern)pattern;
|
|
1890
|
-
if (a.Contains("up")) sp.ScrollVertical(ScrollAmount.SmallDecrement);
|
|
1891
|
-
else if (a.Contains("down")) sp.ScrollVertical(ScrollAmount.SmallIncrement);
|
|
1892
|
-
else if (a.Contains("left")) sp.ScrollHorizontal(ScrollAmount.SmallDecrement);
|
|
1893
|
-
else if (a.Contains("right")) sp.ScrollHorizontal(ScrollAmount.SmallIncrement);
|
|
1894
|
-
else throw new ToolError("bad_action", "unsupported scroll direction in '" + action + "'");
|
|
1895
|
-
}
|
|
1896
|
-
else
|
|
1897
|
-
{
|
|
1898
|
-
throw new ToolError("action_unavailable", "element [" + elementIndex + "] does not expose action '" + action + "'");
|
|
1899
|
-
}
|
|
1900
|
-
|
|
1901
|
-
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1902
|
-
r["performed"] = action;
|
|
1903
|
-
r["element_index"] = elementIndex;
|
|
1904
|
-
return r;
|
|
1905
|
-
}
|
|
1906
|
-
}
|
|
1
|
+
|
|
2
|
+
using System;
|
|
3
|
+
using System.Collections.Generic;
|
|
4
|
+
using System.Diagnostics;
|
|
5
|
+
using System.Drawing;
|
|
6
|
+
using System.Drawing.Imaging;
|
|
7
|
+
using System.IO;
|
|
8
|
+
using System.Runtime.InteropServices;
|
|
9
|
+
using System.Text;
|
|
10
|
+
using System.Windows.Automation;
|
|
11
|
+
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// Platform driver contract. Windows is the only implementation today; the
|
|
14
|
+
// interface exists so macOS (AX + CGEvent) and Linux (AT-SPI) drivers can be
|
|
15
|
+
// added without touching the DSH plugin layer or the stdio protocol.
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
public interface IPlatformDriver : IDisposable
|
|
18
|
+
{
|
|
19
|
+
string PlatformName { get; }
|
|
20
|
+
Dictionary<string, object> Ping(Dictionary<string, object> p);
|
|
21
|
+
Dictionary<string, object> Hello(Dictionary<string, object> p);
|
|
22
|
+
Dictionary<string, object> RestoreCursor(Dictionary<string, object> p);
|
|
23
|
+
void Maintenance();
|
|
24
|
+
Dictionary<string, object> Close(Dictionary<string, object> p);
|
|
25
|
+
Dictionary<string, object> ListWindows(Dictionary<string, object> p);
|
|
26
|
+
Dictionary<string, object> ListApps(Dictionary<string, object> p);
|
|
27
|
+
Dictionary<string, object> GetWindow(Dictionary<string, object> p);
|
|
28
|
+
Dictionary<string, object> LaunchApp(Dictionary<string, object> p);
|
|
29
|
+
Dictionary<string, object> GetWindowState(Dictionary<string, object> p);
|
|
30
|
+
Dictionary<string, object> Click(Dictionary<string, object> p);
|
|
31
|
+
Dictionary<string, object> PressKey(Dictionary<string, object> p);
|
|
32
|
+
Dictionary<string, object> TypeText(Dictionary<string, object> p);
|
|
33
|
+
Dictionary<string, object> Scroll(Dictionary<string, object> p);
|
|
34
|
+
Dictionary<string, object> Drag(Dictionary<string, object> p);
|
|
35
|
+
Dictionary<string, object> SetValue(Dictionary<string, object> p);
|
|
36
|
+
Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p);
|
|
37
|
+
Dictionary<string, object> ActivateWindow(Dictionary<string, object> p);
|
|
38
|
+
Dictionary<string, object> OverlayShow(Dictionary<string, object> p);
|
|
39
|
+
Dictionary<string, object> OverlayHide(Dictionary<string, object> p);
|
|
40
|
+
Dictionary<string, object> Configure(Dictionary<string, object> p);
|
|
41
|
+
Dictionary<string, object> ApproveApp(Dictionary<string, object> p);
|
|
42
|
+
Dictionary<string, object> EndTurn(Dictionary<string, object> p);
|
|
43
|
+
Dictionary<string, object> InvalidateState(Dictionary<string, object> p);
|
|
44
|
+
Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p);
|
|
45
|
+
Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/// Per-request time budget. Codex sends x-oai-cua-request-budget-ms with every call
|
|
49
|
+
/// and the helper honours it; here the budget bounds capture waits and is reported
|
|
50
|
+
/// back when a request overruns so the client and the helper agree on the deadline.
|
|
51
|
+
public static class Budget
|
|
52
|
+
{
|
|
53
|
+
static long _deadlineTicks;
|
|
54
|
+
static string _cancelFile;
|
|
55
|
+
|
|
56
|
+
public static void Begin(long ms, string cancelFile)
|
|
57
|
+
{
|
|
58
|
+
_deadlineTicks = ms > 0 ? DateTime.UtcNow.Ticks + ms * TimeSpan.TicksPerMillisecond : 0;
|
|
59
|
+
_cancelFile = cancelFile;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
public static void Check()
|
|
63
|
+
{
|
|
64
|
+
if (_cancelFile != null && File.Exists(_cancelFile))
|
|
65
|
+
{
|
|
66
|
+
string reason = "cancelled";
|
|
67
|
+
try { reason = File.ReadAllText(_cancelFile).Trim(); } catch { }
|
|
68
|
+
if (reason != "turn_ended" && reason != "timeout") reason = "cancelled";
|
|
69
|
+
throw new ToolError(reason, "the native request was stopped; take a fresh observation before continuing");
|
|
70
|
+
}
|
|
71
|
+
if (Expired()) throw new ToolError("timeout", "the native request exceeded its time budget");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
public static long RemainingMs()
|
|
75
|
+
{
|
|
76
|
+
if (_deadlineTicks == 0) return long.MaxValue / 4;
|
|
77
|
+
return (_deadlineTicks - DateTime.UtcNow.Ticks) / TimeSpan.TicksPerMillisecond;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
public static bool Expired()
|
|
81
|
+
{
|
|
82
|
+
return _deadlineTicks != 0 && RemainingMs() <= 0;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
public class ToolError : Exception
|
|
87
|
+
{
|
|
88
|
+
public readonly string Code;
|
|
89
|
+
public ToolError(string code, string message) : base(message) { Code = code; }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
// Windows driver: Win32 + UI Automation + PrintWindow.
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
public class WindowsDriver : IPlatformDriver
|
|
96
|
+
{
|
|
97
|
+
public string PlatformName { get { return "windows"; } }
|
|
98
|
+
|
|
99
|
+
// ---- win32 ----
|
|
100
|
+
delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
|
|
101
|
+
|
|
102
|
+
[DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc cb, IntPtr lParam);
|
|
103
|
+
[DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
|
|
104
|
+
[DllImport("user32.dll")] static extern bool IsWindow(IntPtr hWnd);
|
|
105
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowTextW(IntPtr hWnd, StringBuilder sb, int max);
|
|
106
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetClassNameW(IntPtr hWnd, StringBuilder sb, int max);
|
|
107
|
+
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
|
|
108
|
+
[DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT r);
|
|
109
|
+
[DllImport("user32.dll")] static extern bool GetClientRect(IntPtr hWnd, out RECT r);
|
|
110
|
+
[DllImport("user32.dll")] static extern bool PrintWindow(IntPtr hWnd, IntPtr hdc, uint flags);
|
|
111
|
+
[DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hWnd);
|
|
112
|
+
[DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hWnd, int cmd);
|
|
113
|
+
[DllImport("user32.dll")] static extern bool IsIconic(IntPtr hWnd);
|
|
114
|
+
[DllImport("user32.dll")] static extern bool SetCursorPos(int x, int y);
|
|
115
|
+
[DllImport("user32.dll")] static extern bool GetCursorPos(out POINT p);
|
|
116
|
+
[DllImport("user32.dll")] static extern uint SendInput(uint n, INPUT[] inputs, int size);
|
|
117
|
+
[DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
|
|
118
|
+
[DllImport("user32.dll")] static extern IntPtr SetFocus(IntPtr hWnd);
|
|
119
|
+
[DllImport("user32.dll")] static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach);
|
|
120
|
+
[DllImport("kernel32.dll")] static extern uint GetCurrentThreadId();
|
|
121
|
+
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid);
|
|
122
|
+
[DllImport("user32.dll")] static extern bool IsWindowEnabled(IntPtr hWnd);
|
|
123
|
+
[DllImport("user32.dll")] static extern bool SystemParametersInfo(uint action, uint param, IntPtr pv, uint winIni);
|
|
124
|
+
[DllImport("user32.dll")] static extern bool SwitchToThisWindow(IntPtr hWnd, bool fAltTab);
|
|
125
|
+
[DllImport("user32.dll")] static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
|
|
126
|
+
[DllImport("user32.dll")] static extern bool BringWindowToTop(IntPtr hWnd);
|
|
127
|
+
[DllImport("user32.dll")] static extern IntPtr SetThreadDpiAwarenessContext(IntPtr ctx);
|
|
128
|
+
[DllImport("user32.dll")] static extern int GetSystemMetrics(int index);
|
|
129
|
+
[DllImport("user32.dll")] static extern bool SetProcessDPIAware();
|
|
130
|
+
[DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT r, int size);
|
|
131
|
+
[DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr hWnd, uint cmd);
|
|
132
|
+
[DllImport("user32.dll")] static extern int GetWindowLong(IntPtr hWnd, int index);
|
|
133
|
+
|
|
134
|
+
[StructLayout(LayoutKind.Sequential)] struct RECT { public int Left, Top, Right, Bottom; }
|
|
135
|
+
[StructLayout(LayoutKind.Sequential)] struct POINT { public int X, Y; }
|
|
136
|
+
|
|
137
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
138
|
+
struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
|
139
|
+
|
|
140
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
141
|
+
struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
|
142
|
+
|
|
143
|
+
[StructLayout(LayoutKind.Explicit)]
|
|
144
|
+
struct INPUTUNION
|
|
145
|
+
{
|
|
146
|
+
[FieldOffset(0)] public MOUSEINPUT mi;
|
|
147
|
+
[FieldOffset(0)] public KEYBDINPUT ki;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
151
|
+
struct INPUT { public uint type; public INPUTUNION u; }
|
|
152
|
+
|
|
153
|
+
const int INPUT_MOUSE = 0;
|
|
154
|
+
const int INPUT_KEYBOARD = 1;
|
|
155
|
+
const uint KEYEVENTF_KEYUP = 0x0002;
|
|
156
|
+
const uint KEYEVENTF_UNICODE = 0x0004;
|
|
157
|
+
const uint KEYEVENTF_EXTENDEDKEY = 0x0001;
|
|
158
|
+
const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
|
|
159
|
+
const uint MOUSEEVENTF_LEFTUP = 0x0004;
|
|
160
|
+
const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
|
|
161
|
+
const uint MOUSEEVENTF_RIGHTUP = 0x0010;
|
|
162
|
+
const uint MOUSEEVENTF_MIDDLEDOWN = 0x0020;
|
|
163
|
+
const uint MOUSEEVENTF_MIDDLEUP = 0x0040;
|
|
164
|
+
const uint MOUSEEVENTF_WHEEL = 0x0800;
|
|
165
|
+
const uint MOUSEEVENTF_HWHEEL = 0x1000;
|
|
166
|
+
const int DWMWA_EXTENDED_FRAME_BOUNDS = 9;
|
|
167
|
+
|
|
168
|
+
// ---- state ----
|
|
169
|
+
readonly object _sync = new object();
|
|
170
|
+
sealed class Snapshot
|
|
171
|
+
{
|
|
172
|
+
public long WindowId;
|
|
173
|
+
public DateTime At;
|
|
174
|
+
public Dictionary<long, AutomationElement> Elements = new Dictionary<long, AutomationElement>();
|
|
175
|
+
public Dictionary<long, string> FallbackLabels = new Dictionary<long, string>();
|
|
176
|
+
public List<object> Summary = new List<object>();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
readonly Dictionary<long, Snapshot> _snapshots = new Dictionary<long, Snapshot>();
|
|
180
|
+
int _snapshotTtlMs = 120000;
|
|
181
|
+
|
|
182
|
+
/// A screenshot region the model was handed, so a later coordinate action can
|
|
183
|
+
/// be interpreted in that exact screenshot space (Codex: screenshotId).
|
|
184
|
+
sealed class ShotRegion
|
|
185
|
+
{
|
|
186
|
+
public long WindowId;
|
|
187
|
+
public int OriginX;
|
|
188
|
+
public int OriginY;
|
|
189
|
+
public int Width;
|
|
190
|
+
public int Height;
|
|
191
|
+
/// Physical pixels per screenshot pixel. 1.0 for DPI-aware targets; >1 when
|
|
192
|
+
/// the target is DPI-virtualized and PrintWindow returns an unscaled render.
|
|
193
|
+
public double ScaleX = 1.0;
|
|
194
|
+
public double ScaleY = 1.0;
|
|
195
|
+
public DateTime At;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// In-use indicator ("DSH 正在使用您的电脑" + whale-girl). Codex draws an animated
|
|
199
|
+
// border and a virtual cursor; this is the DSH equivalent, and Esc is the kill
|
|
200
|
+
// switch: the overlay drops a cancel file that every input path checks first.
|
|
201
|
+
Process _overlay;
|
|
202
|
+
System.Threading.Timer _cursorWatchdog;
|
|
203
|
+
DateTime _overlayUntil = DateTime.MinValue;
|
|
204
|
+
string _overlayCancelFile;
|
|
205
|
+
string _overlayText = "DSH 正在使用您的电脑";
|
|
206
|
+
string _overlayHint = "Esc 取消";
|
|
207
|
+
string _overlayAccent = "#3f7fd8";
|
|
208
|
+
string _cursorFile; // the overlay polls this to draw the virtual cursor
|
|
209
|
+
/// Turn token that the user interrupted with Esc. Codex persists the same
|
|
210
|
+
/// fact as a marker file per conversation+turn; here the token travels with
|
|
211
|
+
/// every input call, so the block ends exactly when the turn does.
|
|
212
|
+
string _interruptedTurn = null;
|
|
213
|
+
/// Apps the plugin approved through the DSH approval seam. Codex passes the same
|
|
214
|
+
/// fact per request as x-oai-cua-approved-app; here the helper is the enforcer.
|
|
215
|
+
readonly HashSet<string> _approvedApps = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
216
|
+
bool _requireAppApproval = false;
|
|
217
|
+
Process _audioProcess;
|
|
218
|
+
string _audioPath;
|
|
219
|
+
string _audioStopFile;
|
|
220
|
+
|
|
221
|
+
readonly Dictionary<string, ShotRegion> _shots = new Dictionary<string, ShotRegion>();
|
|
222
|
+
int _shotTtlMs = 120000;
|
|
223
|
+
int _shotSeq = 0;
|
|
224
|
+
readonly string _shotGeneration = Guid.NewGuid().ToString("N");
|
|
225
|
+
|
|
226
|
+
const uint GW_OWNER = 4;
|
|
227
|
+
const int GWL_STYLE = -16;
|
|
228
|
+
const int WS_POPUP = unchecked((int)0x80000000);
|
|
229
|
+
|
|
230
|
+
public WindowsDriver()
|
|
231
|
+
{
|
|
232
|
+
Maintenance(); // startup repair: a previous session may have died with cursors hidden
|
|
233
|
+
try { SetThreadDpiAwarenessContext(new IntPtr(-4)); } catch { }
|
|
234
|
+
try { SetProcessDPIAware(); } catch { }
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
public void Dispose()
|
|
238
|
+
{
|
|
239
|
+
if (_cursorWatchdog != null) { _cursorWatchdog.Dispose(); _cursorWatchdog = null; }
|
|
240
|
+
StopOverlay();
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// ---- approval plumbing ----
|
|
244
|
+
public Dictionary<string, object> Configure(Dictionary<string, object> p)
|
|
245
|
+
{
|
|
246
|
+
_requireAppApproval = Json.GetBool(p, "requireAppApproval", _requireAppApproval);
|
|
247
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
248
|
+
r["requireAppApproval"] = _requireAppApproval;
|
|
249
|
+
return r;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
public Dictionary<string, object> ApproveApp(Dictionary<string, object> p)
|
|
253
|
+
{
|
|
254
|
+
string app = Json.GetString(p, "app");
|
|
255
|
+
if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "approve_app requires 'app'");
|
|
256
|
+
lock (_sync) { _approvedApps.Add(app); }
|
|
257
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
258
|
+
r["approved"] = app;
|
|
259
|
+
r["count"] = (long)_approvedApps.Count;
|
|
260
|
+
return r;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/// Turn boundary: drop per-turn state (element snapshots, screenshot regions, the
|
|
264
|
+
/// Esc block, the indicator) but keep session-scoped app approvals.
|
|
265
|
+
public Dictionary<string, object> EndTurn(Dictionary<string, object> p)
|
|
266
|
+
{
|
|
267
|
+
string ending = Json.GetString(p, "turnToken");
|
|
268
|
+
if (ending != null) ClearInterrupt(ending);
|
|
269
|
+
lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
|
|
270
|
+
_interruptedTurn = null;
|
|
271
|
+
StopOverlay();
|
|
272
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
273
|
+
r["ended"] = true;
|
|
274
|
+
r["approvedApps"] = (long)_approvedApps.Count;
|
|
275
|
+
return r;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
public Dictionary<string, object> InvalidateState(Dictionary<string, object> p)
|
|
279
|
+
{
|
|
280
|
+
lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
|
|
281
|
+
StopOverlay();
|
|
282
|
+
return new Dictionary<string, object> { { "invalidated", true } };
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
void AssertAppApproved(IntPtr h)
|
|
286
|
+
{
|
|
287
|
+
if (!_requireAppApproval) return;
|
|
288
|
+
uint pid;
|
|
289
|
+
GetWindowThreadProcessId(h, out pid);
|
|
290
|
+
string id = ProcessPathOf(pid);
|
|
291
|
+
lock (_sync) { if (_approvedApps.Contains(id)) return; }
|
|
292
|
+
throw new ToolError("app_not_approved", "the Computer Use policy has not approved '" + id + "' yet. Approve the app through the DSH approval prompt before acting on its windows.");
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ---- audio recording (WASAPI loopback) ----
|
|
296
|
+
public Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p)
|
|
297
|
+
{
|
|
298
|
+
bool envEnabled = Environment.GetEnvironmentVariable("DSH_CU_AUDIO") == "1";
|
|
299
|
+
bool force = Json.GetBool(p, "force", false);
|
|
300
|
+
// Codex treats computer audio as its own app ("computer-audio") and asks for a
|
|
301
|
+
// separate approval before recording; the plugin passes that here.
|
|
302
|
+
bool audioApproved = Json.GetString(p, "approvedApp") == "computer-audio";
|
|
303
|
+
if (!envEnabled && !force && !audioApproved)
|
|
304
|
+
throw new ToolError("audio_not_enabled", "computer audio recording is not enabled. Approve it (app \"computer-audio\") or set DSH_CU_AUDIO=1.");
|
|
305
|
+
if (_audioProcess != null && !_audioProcess.HasExited)
|
|
306
|
+
throw new ToolError("audio_running", "an audio recording is already running; stop it first");
|
|
307
|
+
|
|
308
|
+
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
|
|
309
|
+
string exe = Path.Combine(dir, "audio", "dsh-cu-audio.exe");
|
|
310
|
+
if (!File.Exists(exe)) throw new ToolError("audio_missing", "audio recorder not found at " + exe);
|
|
311
|
+
|
|
312
|
+
long maxMs = Json.GetLong(p, "maxDurationMs", 60000);
|
|
313
|
+
if (maxMs < 500) maxMs = 500;
|
|
314
|
+
if (maxMs > 300000) maxMs = 300000; // Codex caps a single recording at 5 minutes
|
|
315
|
+
_audioPath = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-" + Guid.NewGuid().ToString("N") + ".wav");
|
|
316
|
+
_audioStopFile = Path.Combine(Path.GetTempPath(), "dsh-cu-audio-stop-" + Guid.NewGuid().ToString("N") + ".flag");
|
|
317
|
+
try { if (File.Exists(_audioStopFile)) File.Delete(_audioStopFile); } catch { }
|
|
318
|
+
|
|
319
|
+
ProcessStartInfo psi = new ProcessStartInfo(exe, "--out \"" + _audioPath + "\" --max-ms " + maxMs + " --stop-file \"" + _audioStopFile + "\"");
|
|
320
|
+
psi.UseShellExecute = false;
|
|
321
|
+
psi.CreateNoWindow = true;
|
|
322
|
+
psi.RedirectStandardOutput = true;
|
|
323
|
+
psi.RedirectStandardError = true;
|
|
324
|
+
_audioProcess = Process.Start(psi);
|
|
325
|
+
System.Threading.Thread.Sleep(400);
|
|
326
|
+
if (_audioProcess.HasExited)
|
|
327
|
+
throw new ToolError("audio_not_started", "computer audio recording did not start in time (the recorder exited immediately)");
|
|
328
|
+
|
|
329
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
330
|
+
r["started"] = true;
|
|
331
|
+
r["filepath"] = _audioPath;
|
|
332
|
+
r["maxDurationMs"] = maxMs;
|
|
333
|
+
return r;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
public Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p)
|
|
337
|
+
{
|
|
338
|
+
if (_audioProcess == null) throw new ToolError("audio_not_started", "no audio recording is running");
|
|
339
|
+
try { if (_audioStopFile != null) File.WriteAllText(_audioStopFile, "stop"); } catch { }
|
|
340
|
+
try { _audioProcess.WaitForExit(8000); } catch { }
|
|
341
|
+
try { if (_audioProcess != null && !_audioProcess.HasExited) _audioProcess.Kill(); } catch { }
|
|
342
|
+
|
|
343
|
+
string path = _audioPath;
|
|
344
|
+
_audioProcess = null;
|
|
345
|
+
_audioStopFile = null;
|
|
346
|
+
_audioPath = null;
|
|
347
|
+
|
|
348
|
+
byte[] audio = new byte[0];
|
|
349
|
+
try { if (path != null && File.Exists(path)) audio = File.ReadAllBytes(path); } catch { }
|
|
350
|
+
|
|
351
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
352
|
+
r["filepath"] = path;
|
|
353
|
+
// Codex returns the raw WAV bytes plus a data URL; base64 is the JSON-safe form.
|
|
354
|
+
r["bytes"] = Convert.ToBase64String(audio);
|
|
355
|
+
r["byteLength"] = (long)audio.Length;
|
|
356
|
+
r["data_url"] = audio.Length > 44 ? "data:audio/wav;base64," + Convert.ToBase64String(audio) : "";
|
|
357
|
+
r["ok"] = audio.Length > 44;
|
|
358
|
+
if (audio.Length <= 44) throw new ToolError("audio_not_recorded", "computer audio recording produced no samples");
|
|
359
|
+
return r;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// ---- in-use indicator ----
|
|
363
|
+
static string OverlayExePath()
|
|
364
|
+
{
|
|
365
|
+
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location);
|
|
366
|
+
return Path.Combine(dir ?? ".", "dsh-cu-overlay.exe");
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
public Dictionary<string, object> OverlayShow(Dictionary<string, object> p)
|
|
370
|
+
{
|
|
371
|
+
string text = Json.GetString(p, "text");
|
|
372
|
+
if (!string.IsNullOrEmpty(text)) _overlayText = text;
|
|
373
|
+
string hint = Json.GetString(p, "hint");
|
|
374
|
+
if (!string.IsNullOrEmpty(hint)) _overlayHint = hint;
|
|
375
|
+
string accent = Json.GetString(p, "accent");
|
|
376
|
+
if (!string.IsNullOrEmpty(accent)) _overlayAccent = accent;
|
|
377
|
+
EnsureOverlay(Json.GetLong(p, "ttlMs", 20000));
|
|
378
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
379
|
+
r["visible"] = true;
|
|
380
|
+
r["cancelFile"] = _overlayCancelFile;
|
|
381
|
+
return r;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
public Dictionary<string, object> OverlayHide(Dictionary<string, object> p)
|
|
385
|
+
{
|
|
386
|
+
StopOverlay();
|
|
387
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
388
|
+
r["visible"] = false;
|
|
389
|
+
return r;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
void EnsureOverlay(long ttlMs)
|
|
393
|
+
{
|
|
394
|
+
string exe = OverlayExePath();
|
|
395
|
+
if (!File.Exists(exe)) return;
|
|
396
|
+
string mode = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR");
|
|
397
|
+
if (mode != null && mode.Equals("off", StringComparison.OrdinalIgnoreCase)) { StopOverlay(); return; }
|
|
398
|
+
string ttlEnv = Environment.GetEnvironmentVariable("DSH_CU_INDICATOR_TTL_MS");
|
|
399
|
+
long parsed;
|
|
400
|
+
if (ttlEnv != null && long.TryParse(ttlEnv, out parsed) && parsed >= 1000) ttlMs = parsed;
|
|
401
|
+
if (_overlay != null && !_overlay.HasExited && DateTime.UtcNow < _overlayUntil)
|
|
402
|
+
{
|
|
403
|
+
_overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
StopOverlay();
|
|
407
|
+
_overlayCancelFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cancel-" + Guid.NewGuid().ToString("N") + ".flag");
|
|
408
|
+
if (_cursorFile == null) _cursorFile = Path.Combine(Path.GetTempPath(), "dsh-cu-cursor-" + Guid.NewGuid().ToString("N").Substring(0, 8) + ".flag");
|
|
409
|
+
try { if (File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
|
|
410
|
+
StringBuilder args = new StringBuilder();
|
|
411
|
+
args.Append("--ttl-ms ").Append(ttlMs);
|
|
412
|
+
args.Append(" --cancel-file \"").Append(_overlayCancelFile).Append('"');
|
|
413
|
+
args.Append(" --text \"").Append(_overlayText).Append('"');
|
|
414
|
+
args.Append(" --hint \"").Append(_overlayHint).Append('"');
|
|
415
|
+
args.Append(" --owner-pid ").Append(Process.GetCurrentProcess().Id);
|
|
416
|
+
args.Append(" --accent \"").Append(_overlayAccent).Append('"');
|
|
417
|
+
if (_cursorFile != null) args.Append(" --cursor-file \"").Append(_cursorFile).Append('"');
|
|
418
|
+
// Suppression is the default (the virtual cursor stands in for the real one);
|
|
419
|
+
// DSH_CU_KEEP_CURSOR=1 keeps the real pointer visible instead.
|
|
420
|
+
if (Environment.GetEnvironmentVariable("DSH_CU_KEEP_CURSOR") == "1") args.Append(" --no-suppress-cursor");
|
|
421
|
+
try
|
|
422
|
+
{
|
|
423
|
+
ProcessStartInfo psi = new ProcessStartInfo(exe, args.ToString());
|
|
424
|
+
psi.UseShellExecute = false;
|
|
425
|
+
_overlay = Process.Start(psi);
|
|
426
|
+
_overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
|
|
427
|
+
// Codex runs a dedicated system-cursor-manager process that restores the
|
|
428
|
+
// pointer the moment its parent dies. This timer gives the same guarantee
|
|
429
|
+
// without a second exe: if the overlay vanishes with cursors hidden, the
|
|
430
|
+
// pointer comes back within half a second, with no request needed.
|
|
431
|
+
if (_cursorWatchdog == null)
|
|
432
|
+
_cursorWatchdog = new System.Threading.Timer(delegate { try { Maintenance(); } catch { } }, null, 500, 500);
|
|
433
|
+
}
|
|
434
|
+
catch { _overlay = null; }
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
void StopOverlay()
|
|
438
|
+
{
|
|
439
|
+
try { if (_overlay != null && !_overlay.HasExited) _overlay.Kill(); } catch { }
|
|
440
|
+
// Kill() skips the overlay's exit handlers, which is where it would restore the
|
|
441
|
+
// system cursors. Restore them from here so a terminated overlay can never leave
|
|
442
|
+
// the user without a pointer.
|
|
443
|
+
RestoreCursors();
|
|
444
|
+
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
445
|
+
try { if (_overlayCancelFile != null && File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
|
|
446
|
+
_overlay = null;
|
|
447
|
+
_overlayUntil = DateTime.MinValue;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/// Esc on the overlay drops a cancel file; honour it before any other input.
|
|
451
|
+
const string EscMessage = "Computer Use was stopped by the user with the physical Escape key. Stop your work, do not call further Computer Use tools in this turn, and send a final message noting that the user stopped Computer Use.";
|
|
452
|
+
|
|
453
|
+
void CheckCancel(Dictionary<string, object> p)
|
|
454
|
+
{
|
|
455
|
+
Budget.Check();
|
|
456
|
+
string token = p == null ? null : Json.GetString(p, "turnToken");
|
|
457
|
+
if (token != null && (token == _interruptedTurn || InterruptMarkerExists(token)))
|
|
458
|
+
throw new ToolError("turn_stopped", EscMessage);
|
|
459
|
+
bool overlayExited = _overlay != null && _overlay.HasExited && _overlay.ExitCode == 130;
|
|
460
|
+
bool cancelFile = _overlayCancelFile != null && File.Exists(_overlayCancelFile);
|
|
461
|
+
if (!overlayExited && !cancelFile) return;
|
|
462
|
+
if (cancelFile) { try { File.Delete(_overlayCancelFile); } catch { } }
|
|
463
|
+
_interruptedTurn = token;
|
|
464
|
+
PersistInterrupt(token);
|
|
465
|
+
StopOverlay();
|
|
466
|
+
Program.RequestedExitCode = 130; // Codex exits 130 when the user hits Esc
|
|
467
|
+
throw new ToolError("cancelled_by_user", EscMessage);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const int ProtocolVersion = 2;
|
|
471
|
+
|
|
472
|
+
static string InterruptMarkerPath(string token)
|
|
473
|
+
{
|
|
474
|
+
string root = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "interrupts");
|
|
475
|
+
StringBuilder safe = new StringBuilder();
|
|
476
|
+
if (token != null)
|
|
477
|
+
{
|
|
478
|
+
for (int i = 0; i < token.Length && safe.Length < 64; i++)
|
|
479
|
+
{
|
|
480
|
+
char ch = token[i];
|
|
481
|
+
if (char.IsLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.') safe.Append(ch);
|
|
482
|
+
else if (ch == ':') safe.Append('-');
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
if (safe.Length == 0) safe.Append("turn");
|
|
486
|
+
return Path.Combine(root, safe.ToString() + ".flag");
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
static void PersistInterrupt(string token)
|
|
490
|
+
{
|
|
491
|
+
try
|
|
492
|
+
{
|
|
493
|
+
string path = InterruptMarkerPath(token);
|
|
494
|
+
Directory.CreateDirectory(Path.GetDirectoryName(path));
|
|
495
|
+
File.WriteAllText(path, DateTime.UtcNow.ToString("o"));
|
|
496
|
+
}
|
|
497
|
+
catch { }
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
static bool InterruptMarkerExists(string token)
|
|
501
|
+
{
|
|
502
|
+
try { return File.Exists(InterruptMarkerPath(token)); } catch { return false; }
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
static void ClearInterrupt(string token)
|
|
506
|
+
{
|
|
507
|
+
try { File.Delete(InterruptMarkerPath(token)); } catch { }
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/// Publish the pointer target for the virtual cursor. The overlay reads this
|
|
511
|
+
/// file; a missing file simply means "no cursor to draw".
|
|
512
|
+
void MarkCursor(int x, int y)
|
|
513
|
+
{
|
|
514
|
+
if (_cursorFile == null) return;
|
|
515
|
+
try { File.WriteAllText(_cursorFile, x.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + y.ToString(System.Globalization.CultureInfo.InvariantCulture) + "," + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture)); } catch { }
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/// Codex reports screenLocked instead of failing with an opaque automation error.
|
|
519
|
+
static void AssertUnlocked()
|
|
520
|
+
{
|
|
521
|
+
IntPtr fg = GetForegroundWindow();
|
|
522
|
+
if (fg == IntPtr.Zero) return;
|
|
523
|
+
uint pid;
|
|
524
|
+
GetWindowThreadProcessId(fg, out pid);
|
|
525
|
+
string proc = "";
|
|
526
|
+
try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
|
|
527
|
+
if (proc.Equals("LockApp", StringComparison.OrdinalIgnoreCase) || proc.Equals("LogonUI", StringComparison.OrdinalIgnoreCase))
|
|
528
|
+
throw new ToolError("screen_locked", "the Windows desktop is locked (LockApp is in the foreground), so Computer Use cannot interact with it. Ask the user to unlock the desktop and stop trying until they do.");
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// ---- helpers ----
|
|
532
|
+
static string TitleOf(IntPtr h)
|
|
533
|
+
{
|
|
534
|
+
StringBuilder sb = new StringBuilder(1024);
|
|
535
|
+
GetWindowTextW(h, sb, sb.Capacity);
|
|
536
|
+
return sb.ToString();
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
static string ClassOf(IntPtr h)
|
|
540
|
+
{
|
|
541
|
+
StringBuilder sb = new StringBuilder(256);
|
|
542
|
+
GetClassNameW(h, sb, sb.Capacity);
|
|
543
|
+
return sb.ToString();
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
static RECT BoundsOf(IntPtr h)
|
|
547
|
+
{
|
|
548
|
+
RECT r;
|
|
549
|
+
try
|
|
550
|
+
{
|
|
551
|
+
if (DwmGetWindowAttribute(h, DWMWA_EXTENDED_FRAME_BOUNDS, out r, Marshal.SizeOf(typeof(RECT))) == 0 && r.Right > r.Left)
|
|
552
|
+
return r;
|
|
553
|
+
}
|
|
554
|
+
catch { }
|
|
555
|
+
GetWindowRect(h, out r);
|
|
556
|
+
return r;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
static Dictionary<string, object> RectDict(int x, int y, int w, int h)
|
|
560
|
+
{
|
|
561
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
562
|
+
d["x"] = x; d["y"] = y; d["width"] = w; d["height"] = h;
|
|
563
|
+
return d;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
static string ProcessPathOf(uint pid)
|
|
567
|
+
{
|
|
568
|
+
try
|
|
569
|
+
{
|
|
570
|
+
Process p = Process.GetProcessById((int)pid);
|
|
571
|
+
try { return p.MainModule.FileName; }
|
|
572
|
+
catch { return p.ProcessName; }
|
|
573
|
+
}
|
|
574
|
+
catch { return "unknown"; }
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
static string ProcessNameOf(uint pid)
|
|
578
|
+
{
|
|
579
|
+
try { return Process.GetProcessById((int)pid).ProcessName; }
|
|
580
|
+
catch { return "unknown"; }
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
Dictionary<string, object> AppOf(uint pid)
|
|
584
|
+
{
|
|
585
|
+
Dictionary<string, object> app = new Dictionary<string, object>();
|
|
586
|
+
string full = ProcessPathOf(pid);
|
|
587
|
+
app["id"] = full;
|
|
588
|
+
app["name"] = Path.GetFileNameWithoutExtension(full);
|
|
589
|
+
app["displayName"] = ProcessNameOf(pid);
|
|
590
|
+
app["pid"] = (long)pid;
|
|
591
|
+
try
|
|
592
|
+
{
|
|
593
|
+
Process proc = Process.GetProcessById((int)pid);
|
|
594
|
+
app["lastUsedDate"] = proc.StartTime.ToUniversalTime().ToString("o");
|
|
595
|
+
}
|
|
596
|
+
catch { }
|
|
597
|
+
return app;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
Dictionary<string, object> WindowDict(IntPtr h, uint pid, int z)
|
|
601
|
+
{
|
|
602
|
+
RECT r = BoundsOf(h);
|
|
603
|
+
Dictionary<string, object> w = new Dictionary<string, object>();
|
|
604
|
+
w["id"] = (long)h.ToInt64();
|
|
605
|
+
w["title"] = TitleOf(h);
|
|
606
|
+
w["app"] = AppOf(pid);
|
|
607
|
+
w["bounds"] = RectDict(r.Left, r.Top, r.Right - r.Left, r.Bottom - r.Top);
|
|
608
|
+
w["zIndex"] = z;
|
|
609
|
+
w["className"] = ClassOf(h);
|
|
610
|
+
return w;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
List<IntPtr> EnumerateWindows(bool visibleOnly)
|
|
614
|
+
{
|
|
615
|
+
List<IntPtr> list = new List<IntPtr>();
|
|
616
|
+
EnumWindows(delegate(IntPtr h, IntPtr lp)
|
|
617
|
+
{
|
|
618
|
+
if (!IsWindow(h)) return true;
|
|
619
|
+
if (visibleOnly && !IsWindowVisible(h)) return true;
|
|
620
|
+
uint pid;
|
|
621
|
+
GetWindowThreadProcessId(h, out pid);
|
|
622
|
+
if (pid == 0) return true;
|
|
623
|
+
RECT r;
|
|
624
|
+
GetWindowRect(h, out r);
|
|
625
|
+
if (r.Right - r.Left < 80 || r.Bottom - r.Top < 60) return true;
|
|
626
|
+
string title = TitleOf(h);
|
|
627
|
+
if (title.Length == 0) return true;
|
|
628
|
+
list.Add(h);
|
|
629
|
+
return true;
|
|
630
|
+
}, IntPtr.Zero);
|
|
631
|
+
return list;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// ---- methods ----
|
|
635
|
+
public Dictionary<string, object> Ping(Dictionary<string, object> p)
|
|
636
|
+
{
|
|
637
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
638
|
+
d["platform"] = PlatformName;
|
|
639
|
+
d["pid"] = (long)Process.GetCurrentProcess().Id;
|
|
640
|
+
d["overlayPid"] = _overlay != null && !_overlay.HasExited ? (object)(long)_overlay.Id : null;
|
|
641
|
+
d["ok"] = true;
|
|
642
|
+
d["uia"] = true;
|
|
643
|
+
return d;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/// Run cheaply before every request: if a previous overlay died while it was hiding
|
|
647
|
+
/// the system cursors, put them back. A blank desktop cursor is worse than a stray
|
|
648
|
+
/// visible one, so this never assumes the overlay cleaned up after itself.
|
|
649
|
+
public void Maintenance()
|
|
650
|
+
{
|
|
651
|
+
try
|
|
652
|
+
{
|
|
653
|
+
string flag = OverlayCursorFlagPath();
|
|
654
|
+
if (!File.Exists(flag)) return;
|
|
655
|
+
int pid;
|
|
656
|
+
if (!int.TryParse(File.ReadAllText(flag).Trim(), out pid)) { RestoreCursors(); return; }
|
|
657
|
+
bool alive;
|
|
658
|
+
try { Process.GetProcessById(pid); alive = true; } catch { alive = false; }
|
|
659
|
+
if (!alive) { RestoreCursors(); try { File.Delete(flag); } catch { } }
|
|
660
|
+
}
|
|
661
|
+
catch { }
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
static string OverlayCursorFlagPath()
|
|
665
|
+
{
|
|
666
|
+
return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "cursor-hidden.flag");
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
static void RestoreCursors()
|
|
670
|
+
{
|
|
671
|
+
try { SystemParametersInfo(0x0057, 0, IntPtr.Zero, 0); } catch { }
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
public Dictionary<string, object> RestoreCursor(Dictionary<string, object> p)
|
|
675
|
+
{
|
|
676
|
+
RestoreCursors();
|
|
677
|
+
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
678
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
679
|
+
r["restored"] = true;
|
|
680
|
+
return r;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
public Dictionary<string, object> Hello(Dictionary<string, object> p)
|
|
684
|
+
{
|
|
685
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
686
|
+
d["ok"] = true;
|
|
687
|
+
d["name"] = "dsh-computer-use";
|
|
688
|
+
d["protocolVersion"] = ProtocolVersion;
|
|
689
|
+
d["platform"] = PlatformName;
|
|
690
|
+
List<object> caps = new List<object>();
|
|
691
|
+
foreach (string c in new string[] {
|
|
692
|
+
"window2", "accessibility_tree", "multi_screenshot", "wgc_capture",
|
|
693
|
+
"virtual_cursor", "audio_recording", "turn_interrupt", "app_approval",
|
|
694
|
+
"screenshot_ids", "budgeted_capture" }) caps.Add(c);
|
|
695
|
+
d["capabilities"] = caps;
|
|
696
|
+
return d;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
public Dictionary<string, object> Close(Dictionary<string, object> p)
|
|
700
|
+
{
|
|
701
|
+
Program.RequestedExitCode = 0;
|
|
702
|
+
Program.CloseRequested = true;
|
|
703
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
704
|
+
d["closed"] = true;
|
|
705
|
+
return d;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
public Dictionary<string, object> ListWindows(Dictionary<string, object> p)
|
|
709
|
+
{
|
|
710
|
+
List<object> outList = new List<object>();
|
|
711
|
+
List<IntPtr> hs = EnumerateWindows(true);
|
|
712
|
+
for (int i = 0; i < hs.Count; i++)
|
|
713
|
+
{
|
|
714
|
+
uint pid;
|
|
715
|
+
GetWindowThreadProcessId(hs[i], out pid);
|
|
716
|
+
outList.Add(WindowDict(hs[i], pid, i));
|
|
717
|
+
}
|
|
718
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
719
|
+
d["windows"] = outList;
|
|
720
|
+
d["count"] = outList.Count;
|
|
721
|
+
return d;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
public Dictionary<string, object> ListApps(Dictionary<string, object> p)
|
|
725
|
+
{
|
|
726
|
+
Dictionary<string, List<object>> grouped = new Dictionary<string, List<object>>();
|
|
727
|
+
Dictionary<string, Dictionary<string, object>> appInfo = new Dictionary<string, Dictionary<string, object>>();
|
|
728
|
+
List<IntPtr> hs = EnumerateWindows(true);
|
|
729
|
+
for (int i = 0; i < hs.Count; i++)
|
|
730
|
+
{
|
|
731
|
+
uint pid;
|
|
732
|
+
GetWindowThreadProcessId(hs[i], out pid);
|
|
733
|
+
Dictionary<string, object> app = AppOf(pid);
|
|
734
|
+
string key = Convert.ToString(app["id"]);
|
|
735
|
+
if (!grouped.ContainsKey(key)) { grouped[key] = new List<object>(); appInfo[key] = app; }
|
|
736
|
+
grouped[key].Add(WindowDict(hs[i], pid, i));
|
|
737
|
+
}
|
|
738
|
+
List<object> apps = new List<object>();
|
|
739
|
+
foreach (KeyValuePair<string, List<object>> kv in grouped)
|
|
740
|
+
{
|
|
741
|
+
Dictionary<string, object> a = appInfo[kv.Key];
|
|
742
|
+
a["isRunning"] = true;
|
|
743
|
+
try
|
|
744
|
+
{
|
|
745
|
+
IntPtr sample = IntPtr.Zero;
|
|
746
|
+
List<IntPtr> windowsOfApp = new List<IntPtr>();
|
|
747
|
+
for (int w = 0; w < hs.Count; w++)
|
|
748
|
+
{
|
|
749
|
+
uint wpid;
|
|
750
|
+
GetWindowThreadProcessId(hs[w], out wpid);
|
|
751
|
+
if (Convert.ToString(AppOf(wpid)["id"]) == kv.Key) { sample = hs[w]; break; }
|
|
752
|
+
}
|
|
753
|
+
if (sample != IntPtr.Zero)
|
|
754
|
+
{
|
|
755
|
+
uint spid;
|
|
756
|
+
GetWindowThreadProcessId(sample, out spid);
|
|
757
|
+
a["lastUsedDate"] = Process.GetProcessById((int)spid).StartTime.ToUniversalTime().ToString("o");
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
catch { }
|
|
761
|
+
a["windows"] = kv.Value;
|
|
762
|
+
apps.Add(a);
|
|
763
|
+
}
|
|
764
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
765
|
+
d["apps"] = apps;
|
|
766
|
+
d["count"] = apps.Count;
|
|
767
|
+
return d;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
public Dictionary<string, object> GetWindow(Dictionary<string, object> p)
|
|
771
|
+
{
|
|
772
|
+
IntPtr h = ResolveWindow(p);
|
|
773
|
+
uint pid;
|
|
774
|
+
GetWindowThreadProcessId(h, out pid);
|
|
775
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
776
|
+
d["window"] = WindowDict(h, pid, 0);
|
|
777
|
+
return d;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
public Dictionary<string, object> ActivateWindow(Dictionary<string, object> p)
|
|
781
|
+
{
|
|
782
|
+
IntPtr h = ResolveWindow(p);
|
|
783
|
+
Activate(h);
|
|
784
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
785
|
+
d["activated"] = true;
|
|
786
|
+
d["window"] = WindowDict(h, PidOf(h), 0);
|
|
787
|
+
return d;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
static uint PidOf(IntPtr h) { uint pid; GetWindowThreadProcessId(h, out pid); return pid; }
|
|
791
|
+
|
|
792
|
+
/// Bring a window to the foreground and PROVE it got there. Input is never
|
|
793
|
+
/// injected into a window that is not foreground: SendInput targets whatever
|
|
794
|
+
/// holds focus, so an unproven activation would type into the user's window.
|
|
795
|
+
static void Activate(IntPtr h)
|
|
796
|
+
{
|
|
797
|
+
if (!EnsureForeground(h))
|
|
798
|
+
throw new ToolError("focus_failed", "could not bring window " + h.ToInt64() + " to the foreground, so the input was refused instead of landing in another window. " + DescribeForeground());
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
static bool EnsureForeground(IntPtr h)
|
|
802
|
+
{
|
|
803
|
+
if (!IsWindow(h)) throw new ToolError("window_gone", "window " + h.ToInt64() + " no longer exists; call list_windows again");
|
|
804
|
+
if (!IsWindowEnabled(h)) throw new ToolError("window_disabled", "window " + h.ToInt64() + " is disabled (a modal dialog may be blocking it)");
|
|
805
|
+
if (IsIconic(h)) { ShowWindow(h, 9); System.Threading.Thread.Sleep(180); }
|
|
806
|
+
if (GetForegroundWindow() == h) return true;
|
|
807
|
+
|
|
808
|
+
uint fgThread = GetWindowThreadProcessId(GetForegroundWindow(), IntPtr.Zero);
|
|
809
|
+
uint thisThread = GetCurrentThreadId();
|
|
810
|
+
bool attached = false;
|
|
811
|
+
try
|
|
812
|
+
{
|
|
813
|
+
if (fgThread != 0 && fgThread != thisThread) attached = AttachThreadInput(thisThread, fgThread, true);
|
|
814
|
+
ShowWindow(h, 5); // SW_SHOW
|
|
815
|
+
BringWindowToTop(h);
|
|
816
|
+
SetForegroundWindow(h);
|
|
817
|
+
SetFocus(h);
|
|
818
|
+
}
|
|
819
|
+
finally { if (attached) AttachThreadInput(thisThread, fgThread, false); }
|
|
820
|
+
|
|
821
|
+
if (WaitForeground(h, 10)) return true;
|
|
822
|
+
|
|
823
|
+
// Windows refuses foreground changes from a background process. The
|
|
824
|
+
// documented-looking escape hatches, in order of intrusiveness:
|
|
825
|
+
// 1. SwitchToThisWindow (alt-tab semantics, no key injection)
|
|
826
|
+
// 2. a bare Alt press, which lifts the foreground lock for this call
|
|
827
|
+
try { SwitchToThisWindow(h, true); } catch { }
|
|
828
|
+
if (WaitForeground(h, 10)) return true;
|
|
829
|
+
|
|
830
|
+
keybd_event(0x12, 0, 0, UIntPtr.Zero); // VK_MENU down
|
|
831
|
+
keybd_event(0x12, 0, 0x0002, UIntPtr.Zero); // VK_MENU up
|
|
832
|
+
SetForegroundWindow(h);
|
|
833
|
+
if (WaitForeground(h, 12)) return true;
|
|
834
|
+
return false;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
/// Name the process that currently owns focus: the usual reason a
|
|
838
|
+
/// background host cannot steal it is an always-on-top utility or an
|
|
839
|
+
/// elevated window.
|
|
840
|
+
static string DescribeForeground()
|
|
841
|
+
{
|
|
842
|
+
IntPtr fg = GetForegroundWindow();
|
|
843
|
+
if (fg == IntPtr.Zero) return "No window currently holds the foreground.";
|
|
844
|
+
uint pid;
|
|
845
|
+
GetWindowThreadProcessId(fg, out pid);
|
|
846
|
+
string proc = "unknown";
|
|
847
|
+
try { proc = Process.GetProcessById((int)pid).ProcessName; } catch { }
|
|
848
|
+
string title = TitleOf(fg);
|
|
849
|
+
string cls = ClassOf(fg);
|
|
850
|
+
return "Foreground is held by " + proc + " (pid " + pid + ", class " + cls + (title.Length > 0 ? ", title \"" + Trim(title, 60) + "\"" : "") + "). Bring the target window forward yourself, or close/pause the utility that keeps stealing focus.";
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
static bool WaitForeground(IntPtr h, int attempts)
|
|
854
|
+
{
|
|
855
|
+
for (int i = 0; i < attempts; i++)
|
|
856
|
+
{
|
|
857
|
+
if (GetForegroundWindow() == h) { System.Threading.Thread.Sleep(70); return true; }
|
|
858
|
+
System.Threading.Thread.Sleep(40);
|
|
859
|
+
}
|
|
860
|
+
return GetForegroundWindow() == h;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
static void Slow() { System.Threading.Thread.Sleep(120); }
|
|
864
|
+
|
|
865
|
+
public Dictionary<string, object> LaunchApp(Dictionary<string, object> p)
|
|
866
|
+
{
|
|
867
|
+
string app = Json.GetString(p, "app");
|
|
868
|
+
if (string.IsNullOrEmpty(app)) throw new ToolError("bad_request", "launch_app requires 'app'");
|
|
869
|
+
// Resolution order matches the API contract: an explicit .exe path first,
|
|
870
|
+
// then a packaged-app id (AUMID) through shell:AppsFolder, then a shell URI
|
|
871
|
+
// such as ms-settings:. A bare unknown string is tried verbatim last.
|
|
872
|
+
string resolved = app;
|
|
873
|
+
if (!File.Exists(app) && app.IndexOf('\\') < 0 && !app.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) && app.IndexOf(':') < 0)
|
|
874
|
+
resolved = "shell:AppsFolder\\" + app;
|
|
875
|
+
ProcessStartInfo psi = new ProcessStartInfo();
|
|
876
|
+
psi.FileName = resolved;
|
|
877
|
+
psi.UseShellExecute = true;
|
|
878
|
+
List<object> args = Json.Get(p, "args") as List<object>;
|
|
879
|
+
if (args != null && args.Count > 0)
|
|
880
|
+
{
|
|
881
|
+
StringBuilder argLine = new StringBuilder();
|
|
882
|
+
for (int i = 0; i < args.Count; i++)
|
|
883
|
+
{
|
|
884
|
+
if (i > 0) argLine.Append(' ');
|
|
885
|
+
string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture) ?? "";
|
|
886
|
+
// A quote or newline would let a caller smuggle extra arguments past the
|
|
887
|
+
// single quoting rule below, so refuse instead of guessing.
|
|
888
|
+
if (a.IndexOf('"') >= 0 || a.IndexOf('\n') >= 0 || a.IndexOf('\r') >= 0)
|
|
889
|
+
throw new ToolError("bad_request", "launch_app argument " + i + " contains a quote or newline; pass a single plain argument");
|
|
890
|
+
if (a.Length == 0 || a.IndexOf(' ') >= 0 || a.IndexOf('\t') >= 0)
|
|
891
|
+
argLine.Append('"').Append(a).Append('"');
|
|
892
|
+
else argLine.Append(a);
|
|
893
|
+
}
|
|
894
|
+
psi.Arguments = argLine.ToString();
|
|
895
|
+
}
|
|
896
|
+
Process proc = Process.Start(psi);
|
|
897
|
+
System.Threading.Thread.Sleep(900);
|
|
898
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
899
|
+
d["launched"] = true;
|
|
900
|
+
d["app"] = app;
|
|
901
|
+
try { d["pid"] = (long)proc.Id; } catch { }
|
|
902
|
+
return d;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
IntPtr ResolveWindow(Dictionary<string, object> p)
|
|
906
|
+
{
|
|
907
|
+
Dictionary<string, object> w = Json.AsObject(Json.Get(p, "window"));
|
|
908
|
+
long id = Json.GetLong(w, "id", 0);
|
|
909
|
+
if (id == 0) id = Json.GetLong(p, "windowId", 0);
|
|
910
|
+
if (id == 0) throw new ToolError("bad_request", "a 'window' object with a numeric 'id' is required (use list_windows / list_apps)");
|
|
911
|
+
IntPtr h = new IntPtr(id);
|
|
912
|
+
if (!IsWindow(h)) throw new ToolError("window_gone", "window " + id + " no longer exists; call list_windows again");
|
|
913
|
+
return h;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// ---- observation ----
|
|
917
|
+
public Dictionary<string, object> GetWindowState(Dictionary<string, object> p)
|
|
918
|
+
{
|
|
919
|
+
IntPtr h = ResolveWindow(p);
|
|
920
|
+
AssertAppApproved(h);
|
|
921
|
+
bool includeText = Json.GetBool(p, "include_text", false);
|
|
922
|
+
bool includeShot = Json.GetBool(p, "include_screenshot", false);
|
|
923
|
+
long maxElements = Json.GetLong(p, "maxElements", 260);
|
|
924
|
+
long ttl = Json.GetLong(p, "ttlMs", _snapshotTtlMs);
|
|
925
|
+
|
|
926
|
+
uint pid;
|
|
927
|
+
GetWindowThreadProcessId(h, out pid);
|
|
928
|
+
Dictionary<string, object> result = new Dictionary<string, object>();
|
|
929
|
+
result["window"] = WindowDict(h, pid, 0);
|
|
930
|
+
|
|
931
|
+
// Codex parity: accessibility is null unless include_text was asked for.
|
|
932
|
+
result["accessibility"] = includeText ? (object)BuildAccessibility(h, maxElements, (int)ttl) : null;
|
|
933
|
+
bool blank;
|
|
934
|
+
result["screenshots"] = includeShot ? (object)CaptureScreenshots(h, out blank) : new List<object>();
|
|
935
|
+
return result;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
Dictionary<string, object> BuildAccessibility(IntPtr h, long maxElements, int ttlMs)
|
|
939
|
+
{
|
|
940
|
+
Snapshot snap = new Snapshot();
|
|
941
|
+
snap.WindowId = h.ToInt64();
|
|
942
|
+
snap.At = DateTime.UtcNow;
|
|
943
|
+
|
|
944
|
+
StringBuilder tree = new StringBuilder();
|
|
945
|
+
int index = 0;
|
|
946
|
+
AutomationElement root = null;
|
|
947
|
+
try { root = AutomationElement.FromHandle(h); }
|
|
948
|
+
catch (Exception ex) { throw new ToolError("uia_unavailable", "UI Automation cannot read this window: " + ex.Message); }
|
|
949
|
+
|
|
950
|
+
string focused = null;
|
|
951
|
+
Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
|
|
952
|
+
|
|
953
|
+
if (index == 0)
|
|
954
|
+
{
|
|
955
|
+
// Electron/Canvas apps may not expose a tree until accessibility is requested;
|
|
956
|
+
// retry once after a short delay.
|
|
957
|
+
System.Threading.Thread.Sleep(180);
|
|
958
|
+
try { root = AutomationElement.FromHandle(h); } catch { }
|
|
959
|
+
Walk(root, 0, maxElements, snap, tree, ref index, h, ref focused);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
lock (_sync) { _snapshots[snap.WindowId] = snap; }
|
|
963
|
+
|
|
964
|
+
Dictionary<string, object> state = new Dictionary<string, object>();
|
|
965
|
+
state["tree"] = tree.ToString();
|
|
966
|
+
state["elementCount"] = (long)index;
|
|
967
|
+
state["elements"] = snap.Summary;
|
|
968
|
+
|
|
969
|
+
// Structured context Codex exposes beside the tree. Each is best-effort:
|
|
970
|
+
// an app that does not implement the pattern simply omits the field.
|
|
971
|
+
try
|
|
972
|
+
{
|
|
973
|
+
AutomationElement doc = root.FindFirst(TreeScope.Descendants,
|
|
974
|
+
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Document));
|
|
975
|
+
if (doc != null)
|
|
976
|
+
{
|
|
977
|
+
object dp;
|
|
978
|
+
if (doc.TryGetCurrentPattern(TextPattern.Pattern, out dp))
|
|
979
|
+
{
|
|
980
|
+
string body = ((TextPattern)dp).DocumentRange.GetText(8000);
|
|
981
|
+
if (!string.IsNullOrEmpty(body)) state["document_text"] = body;
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
object focusPattern;
|
|
985
|
+
AutomationElement focusedElement = AutomationElement.FocusedElement;
|
|
986
|
+
if (focusedElement != null && focusedElement.TryGetCurrentPattern(TextPattern.Pattern, out focusPattern))
|
|
987
|
+
{
|
|
988
|
+
System.Windows.Automation.Text.TextPatternRange[] ranges = ((TextPattern)focusPattern).GetSelection();
|
|
989
|
+
if (ranges != null && ranges.Length > 0)
|
|
990
|
+
{
|
|
991
|
+
string selected = ranges[0].GetText(4000);
|
|
992
|
+
if (!string.IsNullOrEmpty(selected)) state["selected_text"] = selected;
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
Condition selectable = new PropertyCondition(AutomationElement.IsSelectionItemPatternAvailableProperty, true);
|
|
996
|
+
AutomationElementCollection selectedItems = root.FindAll(TreeScope.Descendants, selectable);
|
|
997
|
+
List<object> selectedLines = new List<object>();
|
|
998
|
+
for (int i = 0; i < selectedItems.Count && selectedLines.Count < 20; i++)
|
|
999
|
+
{
|
|
1000
|
+
try
|
|
1001
|
+
{
|
|
1002
|
+
object sp;
|
|
1003
|
+
if (selectedItems[i].TryGetCurrentPattern(SelectionItemPattern.Pattern, out sp) && ((SelectionItemPattern)sp).Current.IsSelected)
|
|
1004
|
+
selectedLines.Add(SafeString(selectedItems[i].Current.Name) + " (" + SafeRole(selectedItems[i].Current) + ")");
|
|
1005
|
+
}
|
|
1006
|
+
catch { }
|
|
1007
|
+
}
|
|
1008
|
+
if (selectedLines.Count > 0) state["selected_elements"] = selectedLines;
|
|
1009
|
+
}
|
|
1010
|
+
catch { }
|
|
1011
|
+
state["snapshotAt"] = snap.At.ToString("o");
|
|
1012
|
+
state["ttlMs"] = (long)ttlMs;
|
|
1013
|
+
if (focused != null) state["focused_element"] = focused;
|
|
1014
|
+
return state;
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
void Walk(AutomationElement el, int depth, long maxElements, Snapshot snap, StringBuilder sb, ref int index, IntPtr hwnd, ref string focused)
|
|
1018
|
+
{
|
|
1019
|
+
if (el == null) return;
|
|
1020
|
+
AutomationElementCollection kids;
|
|
1021
|
+
try { kids = el.FindAll(TreeScope.Children, Condition.TrueCondition); }
|
|
1022
|
+
catch { return; }
|
|
1023
|
+
|
|
1024
|
+
for (int i = 0; i < kids.Count; i++)
|
|
1025
|
+
{
|
|
1026
|
+
if (index >= maxElements) return;
|
|
1027
|
+
AutomationElement k;
|
|
1028
|
+
try { k = kids[i]; } catch { continue; }
|
|
1029
|
+
AutomationElement.AutomationElementInformation info;
|
|
1030
|
+
bool offscreen;
|
|
1031
|
+
try { info = k.Current; offscreen = info.IsOffscreen; } catch { continue; }
|
|
1032
|
+
if (offscreen) continue;
|
|
1033
|
+
try
|
|
1034
|
+
{
|
|
1035
|
+
|
|
1036
|
+
long idx = index++;
|
|
1037
|
+
snap.Elements[idx] = k;
|
|
1038
|
+
|
|
1039
|
+
string role = SafeRole(info);
|
|
1040
|
+
string name = SafeString(info.Name);
|
|
1041
|
+
System.Windows.Rect rect = System.Windows.Rect.Empty;
|
|
1042
|
+
try { rect = info.BoundingRectangle; } catch { }
|
|
1043
|
+
bool hasRect = !rect.IsEmpty && rect.Width > 0 && rect.Height > 0;
|
|
1044
|
+
|
|
1045
|
+
Dictionary<string, object> summary = new Dictionary<string, object>();
|
|
1046
|
+
summary["index"] = idx;
|
|
1047
|
+
summary["role"] = role;
|
|
1048
|
+
summary["name"] = name;
|
|
1049
|
+
if (hasRect)
|
|
1050
|
+
{
|
|
1051
|
+
summary["x"] = (int)rect.X;
|
|
1052
|
+
summary["y"] = (int)rect.Y;
|
|
1053
|
+
summary["width"] = (int)rect.Width;
|
|
1054
|
+
summary["height"] = (int)rect.Height;
|
|
1055
|
+
}
|
|
1056
|
+
snap.Summary.Add(summary);
|
|
1057
|
+
|
|
1058
|
+
sb.Append('[').Append(idx).Append("] ").Append(role);
|
|
1059
|
+
if (name.Length > 0) sb.Append(" \"").Append(Trim(name, 120)).Append('"');
|
|
1060
|
+
if (hasRect) sb.Append(" @").Append((int)rect.X).Append(',').Append((int)rect.Y).Append(' ').Append((int)rect.Width).Append('x').Append((int)rect.Height);
|
|
1061
|
+
|
|
1062
|
+
List<string> actions = ActionsOf(k);
|
|
1063
|
+
if (actions.Count > 0) sb.Append(" [").Append(string.Join(",", actions.ToArray())).Append(']');
|
|
1064
|
+
|
|
1065
|
+
StringBuilder val = new StringBuilder();
|
|
1066
|
+
try
|
|
1067
|
+
{
|
|
1068
|
+
object vp;
|
|
1069
|
+
if (k.TryGetCurrentPattern(ValuePattern.Pattern, out vp))
|
|
1070
|
+
{
|
|
1071
|
+
string v = ((ValuePattern)vp).Current.Value;
|
|
1072
|
+
if (!string.IsNullOrEmpty(v)) val.Append(" value=\"").Append(Trim(v, 90)).Append('"');
|
|
1073
|
+
}
|
|
1074
|
+
else
|
|
1075
|
+
{
|
|
1076
|
+
object tp;
|
|
1077
|
+
if (k.TryGetCurrentPattern(TextPattern.Pattern, out tp))
|
|
1078
|
+
{
|
|
1079
|
+
string t = ((TextPattern)tp).DocumentRange.GetText(120);
|
|
1080
|
+
if (!string.IsNullOrEmpty(t)) val.Append(" text=\"").Append(Trim(t.Replace("\r", " ").Replace("\n", " "), 90)).Append('"');
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
catch { }
|
|
1085
|
+
sb.Append(val.ToString());
|
|
1086
|
+
|
|
1087
|
+
try { if (info.HasKeyboardFocus && focused == null) focused = "[" + idx + "] " + role + " \"" + name + "\""; }
|
|
1088
|
+
catch { }
|
|
1089
|
+
|
|
1090
|
+
sb.Append('\n');
|
|
1091
|
+
if (depth < 18) Walk(k, depth + 1, maxElements, snap, sb, ref index, hwnd, ref focused);
|
|
1092
|
+
}
|
|
1093
|
+
catch (Exception) { continue; }
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
static string SafeRole(AutomationElement.AutomationElementInformation info)
|
|
1098
|
+
{
|
|
1099
|
+
try { return info.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
|
|
1100
|
+
catch { return "Unknown"; }
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
static long ToLong(object v) { try { return Convert.ToInt64(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 0L; } }
|
|
1104
|
+
static double ToDouble(object v) { try { return Convert.ToDouble(v, System.Globalization.CultureInfo.InvariantCulture); } catch { return 1.0; } }
|
|
1105
|
+
|
|
1106
|
+
static string SafeString(string s) { return s == null ? "" : s.Replace("\r", " ").Replace("\n", " ").Trim(); }
|
|
1107
|
+
|
|
1108
|
+
static string Trim(string s, int max) { return s.Length <= max ? s : s.Substring(0, max) + "…"; }
|
|
1109
|
+
|
|
1110
|
+
static List<string> ActionsOf(AutomationElement el)
|
|
1111
|
+
{
|
|
1112
|
+
List<string> a = new List<string>();
|
|
1113
|
+
object pattern;
|
|
1114
|
+
try { if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern)) a.Add("Invoke"); } catch { }
|
|
1115
|
+
try { if (el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern)) a.Add("Toggle"); } catch { }
|
|
1116
|
+
try { if (el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern)) a.Add("Select"); } catch { }
|
|
1117
|
+
try { if (el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern)) a.Add("ExpandCollapse"); } catch { }
|
|
1118
|
+
try { if (el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern)) a.Add("Scroll"); } catch { }
|
|
1119
|
+
try { if (el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern)) a.Add("SetValue"); } catch { }
|
|
1120
|
+
return a;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/// Every top-level window that belongs to the same "visual state" as the
|
|
1124
|
+
/// target: the window itself plus owned/transient popups (menus, tooltips,
|
|
1125
|
+
/// combo drop-downs) that a screenshot of the window alone would hide.
|
|
1126
|
+
List<IntPtr> RelatedWindows(IntPtr target)
|
|
1127
|
+
{
|
|
1128
|
+
RECT main = BoundsOf(target);
|
|
1129
|
+
uint targetPid;
|
|
1130
|
+
GetWindowThreadProcessId(target, out targetPid);
|
|
1131
|
+
List<IntPtr> result = new List<IntPtr>();
|
|
1132
|
+
List<IntPtr> all = EnumerateWindows(false);
|
|
1133
|
+
for (int i = 0; i < all.Count; i++)
|
|
1134
|
+
{
|
|
1135
|
+
IntPtr h = all[i];
|
|
1136
|
+
if (h == target) continue;
|
|
1137
|
+
if (!IsWindowVisible(h)) continue;
|
|
1138
|
+
if (IsIconic(h)) continue;
|
|
1139
|
+
uint pid;
|
|
1140
|
+
GetWindowThreadProcessId(h, out pid);
|
|
1141
|
+
if (pid != targetPid) continue;
|
|
1142
|
+
bool owned = GetWindow(h, GW_OWNER) == target;
|
|
1143
|
+
bool popup = (GetWindowLong(h, GWL_STYLE) & WS_POPUP) != 0;
|
|
1144
|
+
if (!owned && !popup) continue;
|
|
1145
|
+
RECT r = BoundsOf(h);
|
|
1146
|
+
bool intersects = r.Left < main.Right && r.Right > main.Left && r.Top < main.Bottom && r.Bottom > main.Top;
|
|
1147
|
+
if (!intersects) continue;
|
|
1148
|
+
result.Add(h);
|
|
1149
|
+
}
|
|
1150
|
+
return result;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
List<object> CaptureScreenshots(IntPtr h, out bool anyBlank)
|
|
1154
|
+
{
|
|
1155
|
+
anyBlank = false;
|
|
1156
|
+
if (IsIconic(h))
|
|
1157
|
+
throw new ToolError("window_minimized", "window " + h.ToInt64() + " is minimized, so there is nothing to observe: call activate_window (or restore it) and observe again. A minimized window must never be captured, because the image would not be the window.");
|
|
1158
|
+
if (!IsWindowVisible(h))
|
|
1159
|
+
throw new ToolError("window_not_visible", "window " + h.ToInt64() + " is not visible; activate it before observing.");
|
|
1160
|
+
|
|
1161
|
+
List<IntPtr> order = EnumerateWindows(true); // z-order, topmost first
|
|
1162
|
+
List<object> shots = new List<object>();
|
|
1163
|
+
List<IntPtr> targets = new List<IntPtr>();
|
|
1164
|
+
targets.Add(h);
|
|
1165
|
+
targets.AddRange(RelatedWindows(h));
|
|
1166
|
+
|
|
1167
|
+
int total = Math.Max(1, order.Count);
|
|
1168
|
+
for (int i = 0; i < targets.Count; i++)
|
|
1169
|
+
{
|
|
1170
|
+
IntPtr w = targets[i];
|
|
1171
|
+
int zIndex = order.IndexOf(w);
|
|
1172
|
+
if (zIndex < 0) zIndex = total - 1;
|
|
1173
|
+
Dictionary<string, object> shot = CaptureWindow(w);
|
|
1174
|
+
shot["zIndex"] = (long)(total - zIndex); // larger = visually above
|
|
1175
|
+
string id = "shot-" + _shotGeneration + "-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
|
|
1176
|
+
shot["id"] = id;
|
|
1177
|
+
RECT r = BoundsOf(w);
|
|
1178
|
+
lock (_sync)
|
|
1179
|
+
{
|
|
1180
|
+
PruneShots();
|
|
1181
|
+
_shots[id] = new ShotRegion
|
|
1182
|
+
{
|
|
1183
|
+
WindowId = h.ToInt64(), OriginX = r.Left, OriginY = r.Top,
|
|
1184
|
+
Width = (int)ToLong(shot["width"]), Height = (int)ToLong(shot["height"]),
|
|
1185
|
+
ScaleX = ToDouble(shot["scaleX"]), ScaleY = ToDouble(shot["scaleY"]),
|
|
1186
|
+
At = DateTime.UtcNow,
|
|
1187
|
+
};
|
|
1188
|
+
}
|
|
1189
|
+
shots.Add(shot);
|
|
1190
|
+
}
|
|
1191
|
+
return shots;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
void PruneShots()
|
|
1195
|
+
{
|
|
1196
|
+
if (_shots.Count <= 48) return;
|
|
1197
|
+
List<string> stale = new List<string>();
|
|
1198
|
+
DateTime cutoff = DateTime.UtcNow.AddMilliseconds(-_shotTtlMs);
|
|
1199
|
+
foreach (KeyValuePair<string, ShotRegion> kv in _shots)
|
|
1200
|
+
if (kv.Value.At < cutoff) stale.Add(kv.Key);
|
|
1201
|
+
for (int i = 0; i < stale.Count; i++) _shots.Remove(stale[i]);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
ShotRegion ResolveShot(IntPtr h, string screenshotId)
|
|
1205
|
+
{
|
|
1206
|
+
if (string.IsNullOrEmpty(screenshotId))
|
|
1207
|
+
throw new ToolError("bad_request", "screenshotId is empty");
|
|
1208
|
+
ShotRegion region;
|
|
1209
|
+
lock (_sync)
|
|
1210
|
+
{
|
|
1211
|
+
if (!_shots.TryGetValue(screenshotId, out region))
|
|
1212
|
+
throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is no longer cached: it must come from the latest get_window_state for this window. Take a fresh screenshot before retrying.");
|
|
1213
|
+
}
|
|
1214
|
+
if (region.WindowId != h.ToInt64())
|
|
1215
|
+
throw new ToolError("screenshot_mismatch", "screenshot '" + screenshotId + "' belongs to another window; take a fresh observation for this window.");
|
|
1216
|
+
if ((DateTime.UtcNow - region.At).TotalMilliseconds > _shotTtlMs)
|
|
1217
|
+
throw new ToolError("stale_screenshot", "screenshot '" + screenshotId + "' is older than the screenshot TTL. Take a fresh screenshot before retrying.");
|
|
1218
|
+
return region;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/// Codex rejects a point that falls outside the screenshot the caller referenced
|
|
1222
|
+
/// ("is outside screenshot bounds"), because that usually means a stale observation.
|
|
1223
|
+
static void AssertPointInsideShot(double x, double y, ShotRegion region, string screenshotId, string what)
|
|
1224
|
+
{
|
|
1225
|
+
if (double.IsNaN(x) || double.IsNaN(y)) return;
|
|
1226
|
+
if (x >= 0 && y >= 0 && x <= region.Width && y <= region.Height) return;
|
|
1227
|
+
throw new ToolError("point_outside_screenshot",
|
|
1228
|
+
what + " (" + x.ToString("0.##") + ", " + y.ToString("0.##") + ") is outside screenshot bounds (0,0)-(" +
|
|
1229
|
+
region.Width + "," + region.Height + ") for '" + screenshotId + "'. Take a fresh screenshot with get_window_state before retrying.");
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
/// Windows.Graphics.Capture through the .NET 8 helper: composed window content,
|
|
1233
|
+
/// occlusion-proof, no DPI halo. Falls back to PrintWindow/screen when absent.
|
|
1234
|
+
static Bitmap TryCaptureWithWgc(IntPtr hwnd)
|
|
1235
|
+
{
|
|
1236
|
+
try
|
|
1237
|
+
{
|
|
1238
|
+
string dir = Path.GetDirectoryName(typeof(WindowsDriver).Assembly.Location) ?? ".";
|
|
1239
|
+
string exe = Path.Combine(dir, "capture", "dsh-cu-capture.exe");
|
|
1240
|
+
if (!File.Exists(exe)) return null;
|
|
1241
|
+
string tmp = Path.Combine(Path.GetTempPath(), "dsh-cu-wgc-" + Guid.NewGuid().ToString("N") + ".png");
|
|
1242
|
+
try
|
|
1243
|
+
{
|
|
1244
|
+
long budget = Budget.RemainingMs();
|
|
1245
|
+
int waitMs = (int)Math.Max(150, Math.Min(1200, budget - 150));
|
|
1246
|
+
ProcessStartInfo psi = new ProcessStartInfo(exe, "--hwnd " + hwnd.ToInt64().ToString() + " --out \"" + tmp + "\" --timeout " + waitMs);
|
|
1247
|
+
psi.UseShellExecute = false;
|
|
1248
|
+
psi.CreateNoWindow = true;
|
|
1249
|
+
psi.RedirectStandardOutput = true;
|
|
1250
|
+
psi.RedirectStandardError = true;
|
|
1251
|
+
using (Process p = Process.Start(psi))
|
|
1252
|
+
{
|
|
1253
|
+
if (p == null) return null;
|
|
1254
|
+
if (!p.WaitForExit(waitMs + 900)) { try { p.Kill(); } catch { } return null; }
|
|
1255
|
+
if (p.ExitCode != 0 || !File.Exists(tmp)) return null;
|
|
1256
|
+
byte[] bytes = File.ReadAllBytes(tmp);
|
|
1257
|
+
using (MemoryStream ms = new MemoryStream(bytes)) return new Bitmap(ms);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
finally { try { if (File.Exists(tmp)) File.Delete(tmp); } catch { } }
|
|
1261
|
+
}
|
|
1262
|
+
catch { return null; }
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
Dictionary<string, object> CaptureWindow(IntPtr h)
|
|
1266
|
+
{
|
|
1267
|
+
RECT r = BoundsOf(h);
|
|
1268
|
+
int w = Math.Max(1, r.Right - r.Left);
|
|
1269
|
+
int ht = Math.Max(1, r.Bottom - r.Top);
|
|
1270
|
+
// Cap pathological sizes so a maximised multi-monitor window cannot blow the
|
|
1271
|
+
// attachment byte budget.
|
|
1272
|
+
const int cap = 2560;
|
|
1273
|
+
double scale = 1.0;
|
|
1274
|
+
if (w > cap || ht > cap)
|
|
1275
|
+
{
|
|
1276
|
+
scale = Math.Min((double)cap / w, (double)cap / ht);
|
|
1277
|
+
w = Math.Max(1, (int)(w * scale));
|
|
1278
|
+
ht = Math.Max(1, (int)(ht * scale));
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
Bitmap wgc = TryCaptureWithWgc(h);
|
|
1282
|
+
if (wgc != null)
|
|
1283
|
+
{
|
|
1284
|
+
Dictionary<string, object> wg = FinishShot(h, wgc, r, true, 1.0, 1.0);
|
|
1285
|
+
return wg;
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
|
|
1289
|
+
bool printed = false;
|
|
1290
|
+
using (Graphics g = Graphics.FromImage(bmp))
|
|
1291
|
+
{
|
|
1292
|
+
IntPtr hdc = g.GetHdc();
|
|
1293
|
+
try { printed = PrintWindow(h, hdc, 2); }
|
|
1294
|
+
finally { g.ReleaseHdc(hdc); }
|
|
1295
|
+
}
|
|
1296
|
+
if (!printed || IsBlank(bmp))
|
|
1297
|
+
{
|
|
1298
|
+
bmp.Dispose();
|
|
1299
|
+
bmp = CaptureFromScreen(r, w, ht);
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
// DPI-virtualized targets render at their own (usually 96 DPI) size while
|
|
1303
|
+
// occupying a larger physical rect, so PrintWindow leaves a black halo.
|
|
1304
|
+
// Crop to the painted region and remember the physical/screenshot ratio so
|
|
1305
|
+
// coordinate actions can map screenshot pixels back to screen pixels.
|
|
1306
|
+
int fullWidth = bmp.Width, fullHeight = bmp.Height;
|
|
1307
|
+
Rectangle painted = PaintedBounds(bmp);
|
|
1308
|
+
if (painted.Width > 16 && painted.Height > 16 && (painted.Width < bmp.Width || painted.Height < bmp.Height))
|
|
1309
|
+
{
|
|
1310
|
+
Bitmap cropped = bmp.Clone(new Rectangle(0, 0, painted.Right, painted.Bottom), bmp.PixelFormat);
|
|
1311
|
+
bmp.Dispose();
|
|
1312
|
+
bmp = cropped;
|
|
1313
|
+
w = bmp.Width;
|
|
1314
|
+
ht = bmp.Height;
|
|
1315
|
+
}
|
|
1316
|
+
double scaleX = (double)fullWidth / Math.Max(1, bmp.Width);
|
|
1317
|
+
double scaleY = (double)fullHeight / Math.Max(1, bmp.Height);
|
|
1318
|
+
|
|
1319
|
+
byte[] png;
|
|
1320
|
+
using (MemoryStream ms = new MemoryStream())
|
|
1321
|
+
{
|
|
1322
|
+
bmp.Save(ms, ImageFormat.Png);
|
|
1323
|
+
png = ms.ToArray();
|
|
1324
|
+
}
|
|
1325
|
+
bmp.Dispose();
|
|
1326
|
+
|
|
1327
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1328
|
+
d["mediaType"] = "image/png";
|
|
1329
|
+
d["base64"] = Convert.ToBase64String(png);
|
|
1330
|
+
d["width"] = (long)w;
|
|
1331
|
+
d["height"] = (long)ht;
|
|
1332
|
+
d["originX"] = (long)r.Left;
|
|
1333
|
+
d["originY"] = (long)r.Top;
|
|
1334
|
+
d["zIndex"] = 1L;
|
|
1335
|
+
d["printed"] = printed;
|
|
1336
|
+
d["scaleX"] = scaleX;
|
|
1337
|
+
d["scaleY"] = scaleY;
|
|
1338
|
+
return d;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
static Dictionary<string, object> FinishShot(IntPtr h, Bitmap bmp, RECT r, bool printed, double scaleX, double scaleY)
|
|
1342
|
+
{
|
|
1343
|
+
int w = bmp.Width, ht = bmp.Height;
|
|
1344
|
+
byte[] png;
|
|
1345
|
+
using (MemoryStream ms = new MemoryStream())
|
|
1346
|
+
{
|
|
1347
|
+
bmp.Save(ms, ImageFormat.Png);
|
|
1348
|
+
png = ms.ToArray();
|
|
1349
|
+
}
|
|
1350
|
+
bmp.Dispose();
|
|
1351
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1352
|
+
d["mediaType"] = "image/png";
|
|
1353
|
+
d["base64"] = Convert.ToBase64String(png);
|
|
1354
|
+
d["width"] = (long)w;
|
|
1355
|
+
d["height"] = (long)ht;
|
|
1356
|
+
d["originX"] = (long)r.Left;
|
|
1357
|
+
d["originY"] = (long)r.Top;
|
|
1358
|
+
d["zIndex"] = 1L;
|
|
1359
|
+
d["printed"] = printed;
|
|
1360
|
+
d["scaleX"] = scaleX;
|
|
1361
|
+
d["scaleY"] = scaleY;
|
|
1362
|
+
return d;
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
static Bitmap CaptureFromScreen(RECT r, int w, int ht)
|
|
1366
|
+
{
|
|
1367
|
+
Bitmap bmp = new Bitmap(w, ht, PixelFormat.Format24bppRgb);
|
|
1368
|
+
using (Graphics g = Graphics.FromImage(bmp))
|
|
1369
|
+
{
|
|
1370
|
+
try { g.CopyFromScreen(r.Left, r.Top, 0, 0, new Size(w, ht), CopyPixelOperation.SourceCopy); }
|
|
1371
|
+
catch { }
|
|
1372
|
+
}
|
|
1373
|
+
return bmp;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
/// Bounding box of everything that is not uniform black, i.e. the region the
|
|
1377
|
+
/// target actually painted. Used to strip the DPI-virtualization halo.
|
|
1378
|
+
static Rectangle PaintedBounds(Bitmap bmp)
|
|
1379
|
+
{
|
|
1380
|
+
int right = 0, bottom = 0;
|
|
1381
|
+
int step = Math.Max(1, Math.Min(bmp.Width, bmp.Height) / 200);
|
|
1382
|
+
for (int y = 0; y < bmp.Height; y += step)
|
|
1383
|
+
{
|
|
1384
|
+
for (int x = 0; x < bmp.Width; x += step)
|
|
1385
|
+
{
|
|
1386
|
+
Color c = bmp.GetPixel(x, y);
|
|
1387
|
+
if (c.R > 8 || c.G > 8 || c.B > 8)
|
|
1388
|
+
{
|
|
1389
|
+
if (x > right) right = x;
|
|
1390
|
+
if (y > bottom) bottom = y;
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
return new Rectangle(0, 0, Math.Min(bmp.Width, right + step), Math.Min(bmp.Height, bottom + step));
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
static bool IsBlank(Bitmap bmp)
|
|
1398
|
+
{
|
|
1399
|
+
int samples = 0, dark = 0;
|
|
1400
|
+
int stepX = Math.Max(1, bmp.Width / 12);
|
|
1401
|
+
int stepY = Math.Max(1, bmp.Height / 12);
|
|
1402
|
+
for (int y = 0; y < bmp.Height; y += stepY)
|
|
1403
|
+
{
|
|
1404
|
+
for (int x = 0; x < bmp.Width; x += stepX)
|
|
1405
|
+
{
|
|
1406
|
+
Color c = bmp.GetPixel(x, y);
|
|
1407
|
+
samples++;
|
|
1408
|
+
if (c.R < 6 && c.G < 6 && c.B < 6) dark++;
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
return samples > 0 && dark == samples;
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
// ---- element resolution ----
|
|
1415
|
+
AutomationElement ElementAt(IntPtr hwnd, long index)
|
|
1416
|
+
{
|
|
1417
|
+
Snapshot snap;
|
|
1418
|
+
lock (_sync)
|
|
1419
|
+
{
|
|
1420
|
+
if (!_snapshots.TryGetValue(hwnd.ToInt64(), out snap))
|
|
1421
|
+
throw new ToolError("stale_snapshot", "no observation for this window yet: call get_window_state with include_text=true before using element_index");
|
|
1422
|
+
}
|
|
1423
|
+
if ((DateTime.UtcNow - snap.At).TotalMilliseconds > _snapshotTtlMs)
|
|
1424
|
+
throw new ToolError("stale_snapshot", "the element snapshot is older than " + _snapshotTtlMs + "ms: call get_window_state again before acting on element indexes");
|
|
1425
|
+
AutomationElement el;
|
|
1426
|
+
if (!snap.Elements.TryGetValue(index, out el))
|
|
1427
|
+
throw new ToolError("bad_index", "element index " + index + " is not part of the latest observation for this window");
|
|
1428
|
+
return el;
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
static Dictionary<string, object> ScreenshotTarget(IntPtr h, AutomationElement el)
|
|
1432
|
+
{
|
|
1433
|
+
RECT obs = BoundsOf(h);
|
|
1434
|
+
System.Windows.Rect r = el.Current.BoundingRectangle;
|
|
1435
|
+
if (r.IsEmpty || r.Width <= 0 || r.Height <= 0)
|
|
1436
|
+
throw new ToolError("no_geometry", "the element has no bounding rectangle; use x/y coordinates instead");
|
|
1437
|
+
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
1438
|
+
d["screenX"] = (int)r.X + (int)(r.Width / 2);
|
|
1439
|
+
d["screenY"] = (int)r.Y + (int)(r.Height / 2);
|
|
1440
|
+
d["originX"] = obs.Left;
|
|
1441
|
+
d["originY"] = obs.Top;
|
|
1442
|
+
return d;
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
// ---- input ----
|
|
1446
|
+
static void SendMouse(uint flags) { SendMouseData(flags, 0); }
|
|
1447
|
+
|
|
1448
|
+
static void SendMouseData(uint flags, int data)
|
|
1449
|
+
{
|
|
1450
|
+
INPUT inp = new INPUT();
|
|
1451
|
+
inp.type = INPUT_MOUSE;
|
|
1452
|
+
inp.u.mi.dx = 0; inp.u.mi.dy = 0;
|
|
1453
|
+
inp.u.mi.mouseData = (uint)data;
|
|
1454
|
+
inp.u.mi.dwFlags = flags;
|
|
1455
|
+
INPUT[] arr = new INPUT[] { inp };
|
|
1456
|
+
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
static void KeyDown(ushort vk) { SendKey(vk, 0, 0); }
|
|
1460
|
+
static void KeyUp(ushort vk) { SendKey(vk, 0, KEYEVENTF_KEYUP); }
|
|
1461
|
+
|
|
1462
|
+
static void SendKey(ushort vk, ushort scan, uint flags)
|
|
1463
|
+
{
|
|
1464
|
+
INPUT inp = new INPUT();
|
|
1465
|
+
inp.type = INPUT_KEYBOARD;
|
|
1466
|
+
inp.u.ki.wVk = vk;
|
|
1467
|
+
inp.u.ki.wScan = scan;
|
|
1468
|
+
inp.u.ki.dwFlags = flags;
|
|
1469
|
+
INPUT[] arr = new INPUT[] { inp };
|
|
1470
|
+
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
static void SendUnicode(char ch, bool up)
|
|
1474
|
+
{
|
|
1475
|
+
INPUT inp = new INPUT();
|
|
1476
|
+
inp.type = INPUT_KEYBOARD;
|
|
1477
|
+
inp.u.ki.wVk = 0;
|
|
1478
|
+
inp.u.ki.wScan = ch;
|
|
1479
|
+
inp.u.ki.dwFlags = KEYEVENTF_UNICODE | (up ? KEYEVENTF_KEYUP : 0);
|
|
1480
|
+
INPUT[] arr = new INPUT[] { inp };
|
|
1481
|
+
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
void ClickAt(int screenX, int screenY, int count, string button, Dictionary<string, object> p)
|
|
1485
|
+
{
|
|
1486
|
+
SetCursorPos(screenX, screenY);
|
|
1487
|
+
System.Threading.Thread.Sleep(35);
|
|
1488
|
+
for (int i = 0; i < Math.Max(1, count); i++)
|
|
1489
|
+
{
|
|
1490
|
+
CheckCancel(p);
|
|
1491
|
+
if (i > 0) System.Threading.Thread.Sleep(60);
|
|
1492
|
+
uint down = MOUSEEVENTF_LEFTDOWN, up = MOUSEEVENTF_LEFTUP;
|
|
1493
|
+
if (button == "r") { down = MOUSEEVENTF_RIGHTDOWN; up = MOUSEEVENTF_RIGHTUP; }
|
|
1494
|
+
else if (button == "m") { down = MOUSEEVENTF_MIDDLEDOWN; up = MOUSEEVENTF_MIDDLEUP; }
|
|
1495
|
+
SendMouse(down);
|
|
1496
|
+
try { System.Threading.Thread.Sleep(22); }
|
|
1497
|
+
finally { SendMouse(up); }
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
public Dictionary<string, object> Click(Dictionary<string, object> p)
|
|
1502
|
+
{
|
|
1503
|
+
CheckCancel(p);
|
|
1504
|
+
AssertUnlocked();
|
|
1505
|
+
EnsureOverlay(8000);
|
|
1506
|
+
IntPtr h = ResolveWindow(p);
|
|
1507
|
+
AssertAppApproved(h);
|
|
1508
|
+
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1509
|
+
string button = NormalizeButton(Json.GetString(p, "mouse_button"));
|
|
1510
|
+
int count = (int)Json.GetLong(p, "click_count", 1);
|
|
1511
|
+
int sx, sy;
|
|
1512
|
+
|
|
1513
|
+
if (elementIndex >= 0)
|
|
1514
|
+
{
|
|
1515
|
+
AutomationElement el = ElementAt(h, elementIndex);
|
|
1516
|
+
Dictionary<string, object> t = ScreenshotTarget(h, el);
|
|
1517
|
+
sx = (int)t["screenX"]; sy = (int)t["screenY"];
|
|
1518
|
+
// Prefer the semantic invoke when the element exposes one and we are
|
|
1519
|
+
// doing a plain left single click: it survives occlusion and does not
|
|
1520
|
+
// depend on pixel geometry.
|
|
1521
|
+
if (count == 1 && button == "l")
|
|
1522
|
+
{
|
|
1523
|
+
object pattern;
|
|
1524
|
+
try
|
|
1525
|
+
{
|
|
1526
|
+
if (el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1527
|
+
{
|
|
1528
|
+
((InvokePattern)pattern).Invoke();
|
|
1529
|
+
Dictionary<string, object> r2 = new Dictionary<string, object>();
|
|
1530
|
+
r2["clicked"] = "invoke"; r2["element_index"] = elementIndex;
|
|
1531
|
+
return r2;
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
catch (Exception) { }
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
else
|
|
1538
|
+
{
|
|
1539
|
+
double x = Json.GetDouble(p, "x", double.NaN);
|
|
1540
|
+
double y = Json.GetDouble(p, "y", double.NaN);
|
|
1541
|
+
if (double.IsNaN(x) || double.IsNaN(y)) throw new ToolError("bad_request", "click needs either 'element_index' or both 'x' and 'y' (window-relative pixels)");
|
|
1542
|
+
string screenshotId = Json.GetString(p, "screenshotId");
|
|
1543
|
+
int originX, originY;
|
|
1544
|
+
double coordScaleX = 1.0, coordScaleY = 1.0;
|
|
1545
|
+
if (!string.IsNullOrEmpty(screenshotId))
|
|
1546
|
+
{
|
|
1547
|
+
ShotRegion region = ResolveShot(h, screenshotId);
|
|
1548
|
+
AssertPointInsideShot(x, y, region, screenshotId, "click");
|
|
1549
|
+
originX = region.OriginX; originY = region.OriginY;
|
|
1550
|
+
coordScaleX = region.ScaleX; coordScaleY = region.ScaleY;
|
|
1551
|
+
}
|
|
1552
|
+
else
|
|
1553
|
+
{
|
|
1554
|
+
RECT bounds = BoundsOf(h);
|
|
1555
|
+
originX = bounds.Left; originY = bounds.Top;
|
|
1556
|
+
}
|
|
1557
|
+
sx = originX + (int)Math.Round(x * coordScaleX);
|
|
1558
|
+
sy = originY + (int)Math.Round(y * coordScaleY);
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
Activate(h); // pointer clicks need a real foreground window
|
|
1562
|
+
MarkCursor(sx, sy);
|
|
1563
|
+
CheckCancel(p);
|
|
1564
|
+
ClickAt(sx, sy, count, button, p);
|
|
1565
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1566
|
+
r["clicked"] = "pointer";
|
|
1567
|
+
r["screenX"] = (long)sx;
|
|
1568
|
+
r["screenY"] = (long)sy;
|
|
1569
|
+
r["clickCount"] = (long)count;
|
|
1570
|
+
return r;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
static string NormalizeButton(string b)
|
|
1574
|
+
{
|
|
1575
|
+
if (string.IsNullOrEmpty(b)) return "l";
|
|
1576
|
+
string s = b.Trim().ToLowerInvariant();
|
|
1577
|
+
if (s == "right" || s == "r") return "r";
|
|
1578
|
+
if (s == "middle" || s == "m") return "m";
|
|
1579
|
+
return "l";
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
public Dictionary<string, object> PressKey(Dictionary<string, object> p)
|
|
1583
|
+
{
|
|
1584
|
+
CheckCancel(p);
|
|
1585
|
+
AssertUnlocked();
|
|
1586
|
+
EnsureOverlay(8000);
|
|
1587
|
+
IntPtr h = ResolveWindow(p);
|
|
1588
|
+
AssertAppApproved(h);
|
|
1589
|
+
string key = Json.GetString(p, "key");
|
|
1590
|
+
if (string.IsNullOrEmpty(key)) throw new ToolError("bad_request", "press_key requires 'key'");
|
|
1591
|
+
Activate(h);
|
|
1592
|
+
Chord(key, p);
|
|
1593
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1594
|
+
r["pressed"] = key;
|
|
1595
|
+
return r;
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
static readonly Dictionary<string, ushort> KeyMap = BuildKeyMap();
|
|
1599
|
+
|
|
1600
|
+
static Dictionary<string, ushort> BuildKeyMap()
|
|
1601
|
+
{
|
|
1602
|
+
Dictionary<string, ushort> m = new Dictionary<string, ushort>(StringComparer.OrdinalIgnoreCase);
|
|
1603
|
+
for (char c = 'a'; c <= 'z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'a'));
|
|
1604
|
+
for (char c = 'A'; c <= 'Z'; c++) m[c.ToString()] = (ushort)(0x41 + (c - 'A'));
|
|
1605
|
+
for (char c = '0'; c <= '9'; c++) m[c.ToString()] = (ushort)(0x30 + (c - '0'));
|
|
1606
|
+
m["return"] = 0x0D; m["enter"] = 0x0D; m["kp_enter"] = 0x0D;
|
|
1607
|
+
m["tab"] = 0x09; m["space"] = 0x20; m["backspace"] = 0x08; m["escape"] = 0x1B; m["esc"] = 0x1B;
|
|
1608
|
+
m["delete"] = 0x2E; m["insert"] = 0x2D; m["home"] = 0x24; m["end"] = 0x23;
|
|
1609
|
+
m["page_up"] = 0x21; m["pageup"] = 0x21; m["prior"] = 0x21; m["page_down"] = 0x22; m["pagedown"] = 0x22; m["next"] = 0x22;
|
|
1610
|
+
m["left"] = 0x25; m["up"] = 0x26; m["right"] = 0x27; m["down"] = 0x28;
|
|
1611
|
+
m["shift_l"] = 0xA0; m["shift"] = 0xA0; m["shift_r"] = 0xA1;
|
|
1612
|
+
m["control_l"] = 0xA2; m["control"] = 0xA2; m["ctrl"] = 0xA2; m["control_r"] = 0xA3;
|
|
1613
|
+
m["alt_l"] = 0xA4; m["alt"] = 0xA4; m["alt_r"] = 0xA5;
|
|
1614
|
+
m["super_l"] = 0x5B; m["win"] = 0x5B; m["meta"] = 0x5B; m["lwin"] = 0x5B; m["super"] = 0x5B;
|
|
1615
|
+
m["super_r"] = 0x5C; m["rwin"] = 0x5C;
|
|
1616
|
+
m["caps_lock"] = 0x14; m["num_lock"] = 0x90; m["scroll_lock"] = 0x91;
|
|
1617
|
+
m["menu"] = 0x5D; m["apps"] = 0x5D; m["print"] = 0x2C; m["pause"] = 0x13;
|
|
1618
|
+
m["period"] = 0xBE; m["greater"] = 0xBE; m["comma"] = 0xBC; m["less"] = 0xBC;
|
|
1619
|
+
m["slash"] = 0xBF; m["question"] = 0xBF; m["backslash"] = 0xDC; m["bar"] = 0xDC;
|
|
1620
|
+
m["semicolon"] = 0xBA; m["colon"] = 0xBA; m["apostrophe"] = 0xDE; m["quote"] = 0xDE;
|
|
1621
|
+
m["minus"] = 0xBD; m["underscore"] = 0xBD; m["equal"] = 0xBB; m["plus"] = 0xBB;
|
|
1622
|
+
m["bracketleft"] = 0xDB; m["bracketright"] = 0xDD; m["grave"] = 0xC0; m["asciitilde"] = 0xC0;
|
|
1623
|
+
for (int i = 1; i <= 24; i++) { m["f" + i] = (ushort)(0x6F + i); }
|
|
1624
|
+
// Numpad: both X keysym style (KP_0) and the aliases the API documents.
|
|
1625
|
+
for (int i = 0; i <= 9; i++)
|
|
1626
|
+
{
|
|
1627
|
+
m["kp_" + i] = (ushort)(0x60 + i);
|
|
1628
|
+
m["numpad_" + i] = (ushort)(0x60 + i);
|
|
1629
|
+
}
|
|
1630
|
+
m["kp_add"] = 0x6B; m["numpad_add"] = 0x6B; m["plus_numpad"] = 0x6B;
|
|
1631
|
+
m["kp_subtract"] = 0x6D; m["numpad_subtract"] = 0x6D; m["minus_numpad"] = 0x6D;
|
|
1632
|
+
m["kp_multiply"] = 0x6A; m["numpad_multiply"] = 0x6A; m["kp_asterisk"] = 0x6A;
|
|
1633
|
+
m["kp_divide"] = 0x6F; m["numpad_divide"] = 0x6F; m["kp_slash"] = 0x6F;
|
|
1634
|
+
m["kp_decimal"] = 0x6E; m["numpad_decimal"] = 0x6E; m["kp_separator"] = 0x6C;
|
|
1635
|
+
m["numpad_enter"] = 0x0D;
|
|
1636
|
+
return m;
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
static readonly HashSet<ushort> ExtendedKeys = new HashSet<ushort>(new ushort[] {
|
|
1640
|
+
0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x2D,0x2E,0x5B,0x5C,0x5D,0x90,0x6F,0xA3,0xA5
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1643
|
+
static void SendVk(ushort vk, bool up)
|
|
1644
|
+
{
|
|
1645
|
+
uint flags = up ? KEYEVENTF_KEYUP : 0;
|
|
1646
|
+
if (ExtendedKeys.Contains(vk)) flags |= KEYEVENTF_EXTENDEDKEY;
|
|
1647
|
+
SendKey(vk, 0, flags);
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
void Chord(string chord, Dictionary<string, object> p)
|
|
1651
|
+
{
|
|
1652
|
+
string[] parts = chord.Split('+');
|
|
1653
|
+
List<ushort> keys = new List<ushort>();
|
|
1654
|
+
foreach (string raw in parts)
|
|
1655
|
+
{
|
|
1656
|
+
string token = raw.Trim();
|
|
1657
|
+
if (token.Length == 0) continue;
|
|
1658
|
+
keys.Add(ResolveKey(token));
|
|
1659
|
+
}
|
|
1660
|
+
if (keys.Count == 0) throw new ToolError("bad_request", "empty key chord");
|
|
1661
|
+
int pressed = 0;
|
|
1662
|
+
try
|
|
1663
|
+
{
|
|
1664
|
+
for (int i = 0; i < keys.Count; i++)
|
|
1665
|
+
{
|
|
1666
|
+
CheckCancel(p);
|
|
1667
|
+
SendVk(keys[i], false); pressed++;
|
|
1668
|
+
System.Threading.Thread.Sleep(12);
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
finally { for (int i = pressed - 1; i >= 0; i--) SendVk(keys[i], true); }
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
/// The Windows/Meta key family is hard-denied: it opens Run, Settings and
|
|
1675
|
+
/// shell surfaces that Computer Use must never drive. Enforced in the helper
|
|
1676
|
+
/// as well as in the plugin, so a direct protocol call cannot bypass it.
|
|
1677
|
+
static readonly HashSet<string> DeniedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
|
|
1678
|
+
"win", "win_l", "win_r", "lwin", "rwin", "windows", "windows_l", "windows_r",
|
|
1679
|
+
"super", "super_l", "super_r", "meta", "meta_l", "meta_r", "cmd", "command", "os", "os_l", "os_r"
|
|
1680
|
+
};
|
|
1681
|
+
|
|
1682
|
+
static ushort ResolveKey(string token)
|
|
1683
|
+
{
|
|
1684
|
+
if (DeniedKeys.Contains(token))
|
|
1685
|
+
throw new ToolError("hard_denied", "the Windows/Meta/Super key ('" + token + "') is hard-denied: it opens Run, Settings and shell surfaces that Computer Use must never drive.");
|
|
1686
|
+
ushort vk;
|
|
1687
|
+
if (KeyMap.TryGetValue(token, out vk)) return vk;
|
|
1688
|
+
if (token.Length == 1)
|
|
1689
|
+
{
|
|
1690
|
+
char c = token[0];
|
|
1691
|
+
short scan = VkKeyScan(c);
|
|
1692
|
+
if (scan != -1) return (ushort)(scan & 0xFF);
|
|
1693
|
+
return (ushort)char.ToUpperInvariant(c);
|
|
1694
|
+
}
|
|
1695
|
+
throw new ToolError("bad_key", "unknown key name '" + token + "'");
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
[DllImport("user32.dll")] static extern short VkKeyScan(char ch);
|
|
1699
|
+
|
|
1700
|
+
public Dictionary<string, object> TypeText(Dictionary<string, object> p)
|
|
1701
|
+
{
|
|
1702
|
+
CheckCancel(p);
|
|
1703
|
+
AssertUnlocked();
|
|
1704
|
+
EnsureOverlay(8000);
|
|
1705
|
+
IntPtr h = ResolveWindow(p);
|
|
1706
|
+
AssertAppApproved(h);
|
|
1707
|
+
string text = Json.GetString(p, "text");
|
|
1708
|
+
if (text == null) throw new ToolError("bad_request", "type_text requires 'text'");
|
|
1709
|
+
Activate(h);
|
|
1710
|
+
for (int i = 0; i < text.Length; i++)
|
|
1711
|
+
{
|
|
1712
|
+
CheckCancel(p);
|
|
1713
|
+
char c = text[i];
|
|
1714
|
+
if (c == '\n') { SendVk(0x0D, false); SendVk(0x0D, true); continue; }
|
|
1715
|
+
if (c == '\t') { SendVk(0x09, false); SendVk(0x09, true); continue; }
|
|
1716
|
+
if (c == '\r') continue;
|
|
1717
|
+
SendUnicode(c, false);
|
|
1718
|
+
SendUnicode(c, true);
|
|
1719
|
+
if ((i & 15) == 0) System.Threading.Thread.Sleep(4);
|
|
1720
|
+
}
|
|
1721
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1722
|
+
r["typed"] = (long)text.Length;
|
|
1723
|
+
return r;
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
public Dictionary<string, object> Scroll(Dictionary<string, object> p)
|
|
1727
|
+
{
|
|
1728
|
+
CheckCancel(p);
|
|
1729
|
+
AssertUnlocked();
|
|
1730
|
+
EnsureOverlay(8000);
|
|
1731
|
+
IntPtr h = ResolveWindow(p);
|
|
1732
|
+
AssertAppApproved(h);
|
|
1733
|
+
// Validation first: an invalid call must not activate a window or move the pointer.
|
|
1734
|
+
double scrollY = Json.GetDouble(p, "scrollY", 0);
|
|
1735
|
+
double scrollX = Json.GetDouble(p, "scrollX", 0);
|
|
1736
|
+
double x = Json.GetDouble(p, "x", double.NaN);
|
|
1737
|
+
double y = Json.GetDouble(p, "y", double.NaN);
|
|
1738
|
+
if (double.IsNaN(x) || double.IsNaN(y))
|
|
1739
|
+
throw new ToolError("bad_request", "scroll requires finite 'x' and 'y' (the point to scroll from, in window/screenshot coordinates); click a pane first if it needs focus");
|
|
1740
|
+
if (Math.Abs(scrollX) < 0.0001 && Math.Abs(scrollY) < 0.0001)
|
|
1741
|
+
throw new ToolError("bad_request", "scroll requires a non-zero scrollX or scrollY");
|
|
1742
|
+
Activate(h);
|
|
1743
|
+
int sx, sy;
|
|
1744
|
+
string scrollShot = Json.GetString(p, "screenshotId");
|
|
1745
|
+
int originLeft, originTop;
|
|
1746
|
+
double scrollScaleX = 1.0, scrollScaleY = 1.0;
|
|
1747
|
+
if (!string.IsNullOrEmpty(scrollShot))
|
|
1748
|
+
{
|
|
1749
|
+
ShotRegion region = ResolveShot(h, scrollShot);
|
|
1750
|
+
AssertPointInsideShot(x, y, region, scrollShot, "scroll origin");
|
|
1751
|
+
originLeft = region.OriginX; originTop = region.OriginY;
|
|
1752
|
+
scrollScaleX = region.ScaleX; scrollScaleY = region.ScaleY;
|
|
1753
|
+
}
|
|
1754
|
+
else
|
|
1755
|
+
{
|
|
1756
|
+
RECT b = BoundsOf(h);
|
|
1757
|
+
originLeft = b.Left; originTop = b.Top;
|
|
1758
|
+
}
|
|
1759
|
+
sx = originLeft + (int)Math.Round(x * scrollScaleX);
|
|
1760
|
+
sy = originTop + (int)Math.Round(y * scrollScaleY);
|
|
1761
|
+
MarkCursor(sx, sy);
|
|
1762
|
+
SetCursorPos(sx, sy);
|
|
1763
|
+
System.Threading.Thread.Sleep(40);
|
|
1764
|
+
if (Math.Abs(scrollY) > 0.0001) SendMouseData(MOUSEEVENTF_WHEEL, (int)(-scrollY * 120));
|
|
1765
|
+
if (Math.Abs(scrollX) > 0.0001) SendMouseData(MOUSEEVENTF_HWHEEL, (int)(-scrollX * 120));
|
|
1766
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1767
|
+
r["scrolled"] = true;
|
|
1768
|
+
r["screenX"] = (long)sx; r["screenY"] = (long)sy;
|
|
1769
|
+
return r;
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
public Dictionary<string, object> Drag(Dictionary<string, object> p)
|
|
1773
|
+
{
|
|
1774
|
+
CheckCancel(p);
|
|
1775
|
+
AssertUnlocked();
|
|
1776
|
+
EnsureOverlay(8000);
|
|
1777
|
+
IntPtr h = ResolveWindow(p);
|
|
1778
|
+
AssertAppApproved(h);
|
|
1779
|
+
double fromX = Json.GetDouble(p, "from_x", double.NaN);
|
|
1780
|
+
double fromY = Json.GetDouble(p, "from_y", double.NaN);
|
|
1781
|
+
double toX = Json.GetDouble(p, "to_x", double.NaN);
|
|
1782
|
+
double toY = Json.GetDouble(p, "to_y", double.NaN);
|
|
1783
|
+
if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
|
|
1784
|
+
throw new ToolError("bad_request", "drag requires finite from_x/from_y/to_x/to_y (window-relative pixels)");
|
|
1785
|
+
Activate(h);
|
|
1786
|
+
int originX, originY;
|
|
1787
|
+
double dragScaleX = 1.0, dragScaleY = 1.0;
|
|
1788
|
+
string dragShot = Json.GetString(p, "screenshotId");
|
|
1789
|
+
if (!string.IsNullOrEmpty(dragShot))
|
|
1790
|
+
{
|
|
1791
|
+
ShotRegion region = ResolveShot(h, dragShot);
|
|
1792
|
+
AssertPointInsideShot(fromX, fromY, region, dragShot, "drag start");
|
|
1793
|
+
AssertPointInsideShot(toX, toY, region, dragShot, "drag end");
|
|
1794
|
+
originX = region.OriginX; originY = region.OriginY;
|
|
1795
|
+
dragScaleX = region.ScaleX; dragScaleY = region.ScaleY;
|
|
1796
|
+
}
|
|
1797
|
+
else
|
|
1798
|
+
{
|
|
1799
|
+
RECT b = BoundsOf(h);
|
|
1800
|
+
originX = b.Left; originY = b.Top;
|
|
1801
|
+
}
|
|
1802
|
+
int fx = originX + (int)Math.Round(fromX * dragScaleX);
|
|
1803
|
+
int fy = originY + (int)Math.Round(fromY * dragScaleY);
|
|
1804
|
+
int tx = originX + (int)Math.Round(toX * dragScaleX);
|
|
1805
|
+
int ty = originY + (int)Math.Round(toY * dragScaleY);
|
|
1806
|
+
|
|
1807
|
+
MarkCursor(fx, fy);
|
|
1808
|
+
SetCursorPos(fx, fy);
|
|
1809
|
+
System.Threading.Thread.Sleep(60);
|
|
1810
|
+
CheckCancel(p);
|
|
1811
|
+
SendMouse(MOUSEEVENTF_LEFTDOWN);
|
|
1812
|
+
try
|
|
1813
|
+
{
|
|
1814
|
+
System.Threading.Thread.Sleep(80);
|
|
1815
|
+
const int steps = 14;
|
|
1816
|
+
for (int i = 1; i <= steps; i++)
|
|
1817
|
+
{
|
|
1818
|
+
CheckCancel(p);
|
|
1819
|
+
int x = fx + (tx - fx) * i / steps;
|
|
1820
|
+
int y = fy + (ty - fy) * i / steps;
|
|
1821
|
+
SetCursorPos(x, y);
|
|
1822
|
+
System.Threading.Thread.Sleep(18);
|
|
1823
|
+
}
|
|
1824
|
+
System.Threading.Thread.Sleep(90);
|
|
1825
|
+
CheckCancel(p);
|
|
1826
|
+
}
|
|
1827
|
+
finally { SendMouse(MOUSEEVENTF_LEFTUP); }
|
|
1828
|
+
|
|
1829
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1830
|
+
r["dragged"] = true;
|
|
1831
|
+
r["from"] = new long[] { fx, fy };
|
|
1832
|
+
r["to"] = new long[] { tx, ty };
|
|
1833
|
+
return r;
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
public Dictionary<string, object> SetValue(Dictionary<string, object> p)
|
|
1837
|
+
{
|
|
1838
|
+
CheckCancel(p);
|
|
1839
|
+
AssertUnlocked();
|
|
1840
|
+
EnsureOverlay(8000);
|
|
1841
|
+
IntPtr h = ResolveWindow(p);
|
|
1842
|
+
AssertAppApproved(h);
|
|
1843
|
+
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1844
|
+
if (elementIndex < 0) throw new ToolError("bad_request", "set_value requires 'element_index'");
|
|
1845
|
+
string value = Json.GetString(p, "value");
|
|
1846
|
+
if (value == null) throw new ToolError("bad_request", "set_value requires 'value'");
|
|
1847
|
+
AutomationElement el = ElementAt(h, elementIndex);
|
|
1848
|
+
object pattern;
|
|
1849
|
+
if (!el.TryGetCurrentPattern(ValuePattern.Pattern, out pattern))
|
|
1850
|
+
throw new ToolError("pattern_unavailable", "element [" + elementIndex + "] does not expose the Value pattern; use click + type_text instead");
|
|
1851
|
+
((ValuePattern)pattern).SetValue(value);
|
|
1852
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1853
|
+
r["valueSet"] = true;
|
|
1854
|
+
r["element_index"] = elementIndex;
|
|
1855
|
+
return r;
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
public Dictionary<string, object> PerformSecondaryAction(Dictionary<string, object> p)
|
|
1859
|
+
{
|
|
1860
|
+
CheckCancel(p);
|
|
1861
|
+
AssertUnlocked();
|
|
1862
|
+
EnsureOverlay(8000);
|
|
1863
|
+
IntPtr h = ResolveWindow(p);
|
|
1864
|
+
AssertAppApproved(h);
|
|
1865
|
+
long elementIndex = Json.GetLong(p, "element_index", -1);
|
|
1866
|
+
string action = Json.GetString(p, "action");
|
|
1867
|
+
if (elementIndex < 0 || string.IsNullOrEmpty(action))
|
|
1868
|
+
throw new ToolError("bad_request", "perform_secondary_action requires 'element_index' and 'action'");
|
|
1869
|
+
AutomationElement el = ElementAt(h, elementIndex);
|
|
1870
|
+
string a = action.Trim().ToLowerInvariant();
|
|
1871
|
+
object pattern;
|
|
1872
|
+
|
|
1873
|
+
if ((a == "raise" || a == "focus") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1874
|
+
{ ((InvokePattern)pattern).Invoke(); }
|
|
1875
|
+
else if (a == "raise" || a == "focus") { try { el.SetFocus(); } catch (Exception ex) { throw new ToolError("action_failed", ex.Message); } }
|
|
1876
|
+
else if ((a == "invoke" || a == "press") && el.TryGetCurrentPattern(InvokePattern.Pattern, out pattern))
|
|
1877
|
+
{ ((InvokePattern)pattern).Invoke(); }
|
|
1878
|
+
else if ((a == "expand" || a == "collapse") && el.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out pattern))
|
|
1879
|
+
{
|
|
1880
|
+
ExpandCollapsePattern ec = (ExpandCollapsePattern)pattern;
|
|
1881
|
+
if (a == "expand") ec.Expand(); else ec.Collapse();
|
|
1882
|
+
}
|
|
1883
|
+
else if ((a == "toggle") && el.TryGetCurrentPattern(TogglePattern.Pattern, out pattern))
|
|
1884
|
+
{ ((TogglePattern)pattern).Toggle(); }
|
|
1885
|
+
else if ((a == "select" || a == "selectitem") && el.TryGetCurrentPattern(SelectionItemPattern.Pattern, out pattern))
|
|
1886
|
+
{ ((SelectionItemPattern)pattern).Select(); }
|
|
1887
|
+
else if (a.StartsWith("scroll") && el.TryGetCurrentPattern(ScrollPattern.Pattern, out pattern))
|
|
1888
|
+
{
|
|
1889
|
+
ScrollPattern sp = (ScrollPattern)pattern;
|
|
1890
|
+
if (a.Contains("up")) sp.ScrollVertical(ScrollAmount.SmallDecrement);
|
|
1891
|
+
else if (a.Contains("down")) sp.ScrollVertical(ScrollAmount.SmallIncrement);
|
|
1892
|
+
else if (a.Contains("left")) sp.ScrollHorizontal(ScrollAmount.SmallDecrement);
|
|
1893
|
+
else if (a.Contains("right")) sp.ScrollHorizontal(ScrollAmount.SmallIncrement);
|
|
1894
|
+
else throw new ToolError("bad_action", "unsupported scroll direction in '" + action + "'");
|
|
1895
|
+
}
|
|
1896
|
+
else
|
|
1897
|
+
{
|
|
1898
|
+
throw new ToolError("action_unavailable", "element [" + elementIndex + "] does not expose action '" + action + "'");
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1902
|
+
r["performed"] = action;
|
|
1903
|
+
r["element_index"] = elementIndex;
|
|
1904
|
+
return r;
|
|
1905
|
+
}
|
|
1906
|
+
}
|