deepseek-computer-use 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +89 -29
- package/assets/cover-whale-girl.png +0 -0
- package/lib/bridge.js +153 -150
- package/lib/build-helper.mjs +18 -4
- package/lib/build.mjs +2 -2
- package/lib/index.js +53 -34
- package/lib/parameters.js +50 -0
- package/lib/safety.js +2 -9
- package/lib/tools.js +131 -32
- package/native/Driver.cs +145 -42
- package/native/Overlay.cs +92 -29
- package/native/Program.cs +6 -1
- package/package.json +8 -4
- package/skills/computer-use/SKILL.md +5 -2
package/native/Driver.cs
CHANGED
|
@@ -19,6 +19,8 @@ public interface IPlatformDriver : IDisposable
|
|
|
19
19
|
string PlatformName { get; }
|
|
20
20
|
Dictionary<string, object> Ping(Dictionary<string, object> p);
|
|
21
21
|
Dictionary<string, object> Hello(Dictionary<string, object> p);
|
|
22
|
+
Dictionary<string, object> RestoreCursor(Dictionary<string, object> p);
|
|
23
|
+
void Maintenance();
|
|
22
24
|
Dictionary<string, object> Close(Dictionary<string, object> p);
|
|
23
25
|
Dictionary<string, object> ListWindows(Dictionary<string, object> p);
|
|
24
26
|
Dictionary<string, object> ListApps(Dictionary<string, object> p);
|
|
@@ -38,6 +40,7 @@ public interface IPlatformDriver : IDisposable
|
|
|
38
40
|
Dictionary<string, object> Configure(Dictionary<string, object> p);
|
|
39
41
|
Dictionary<string, object> ApproveApp(Dictionary<string, object> p);
|
|
40
42
|
Dictionary<string, object> EndTurn(Dictionary<string, object> p);
|
|
43
|
+
Dictionary<string, object> InvalidateState(Dictionary<string, object> p);
|
|
41
44
|
Dictionary<string, object> StartAudioRecording(Dictionary<string, object> p);
|
|
42
45
|
Dictionary<string, object> StopAudioRecording(Dictionary<string, object> p);
|
|
43
46
|
}
|
|
@@ -48,10 +51,24 @@ public interface IPlatformDriver : IDisposable
|
|
|
48
51
|
public static class Budget
|
|
49
52
|
{
|
|
50
53
|
static long _deadlineTicks;
|
|
54
|
+
static string _cancelFile;
|
|
51
55
|
|
|
52
|
-
public static void Begin(long ms)
|
|
56
|
+
public static void Begin(long ms, string cancelFile)
|
|
53
57
|
{
|
|
54
58
|
_deadlineTicks = ms > 0 ? DateTime.UtcNow.Ticks + ms * TimeSpan.TicksPerMillisecond : 0;
|
|
59
|
+
_cancelFile = cancelFile;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
public static void Check()
|
|
63
|
+
{
|
|
64
|
+
if (_cancelFile != null && File.Exists(_cancelFile))
|
|
65
|
+
{
|
|
66
|
+
string reason = "cancelled";
|
|
67
|
+
try { reason = File.ReadAllText(_cancelFile).Trim(); } catch { }
|
|
68
|
+
if (reason != "turn_ended" && reason != "timeout") reason = "cancelled";
|
|
69
|
+
throw new ToolError(reason, "the native request was stopped; take a fresh observation before continuing");
|
|
70
|
+
}
|
|
71
|
+
if (Expired()) throw new ToolError("timeout", "the native request exceeded its time budget");
|
|
55
72
|
}
|
|
56
73
|
|
|
57
74
|
public static long RemainingMs()
|
|
@@ -182,14 +199,12 @@ public class WindowsDriver : IPlatformDriver
|
|
|
182
199
|
// border and a virtual cursor; this is the DSH equivalent, and Esc is the kill
|
|
183
200
|
// switch: the overlay drops a cancel file that every input path checks first.
|
|
184
201
|
Process _overlay;
|
|
202
|
+
System.Threading.Timer _cursorWatchdog;
|
|
185
203
|
DateTime _overlayUntil = DateTime.MinValue;
|
|
186
204
|
string _overlayCancelFile;
|
|
187
|
-
string _overlayImage = "";
|
|
188
|
-
string _overlayCrop = "";
|
|
189
205
|
string _overlayText = "DSH 正在使用您的电脑";
|
|
190
206
|
string _overlayHint = "Esc 取消";
|
|
191
207
|
string _overlayAccent = "#3f7fd8";
|
|
192
|
-
string _overlayCorner = "br";
|
|
193
208
|
string _cursorFile; // the overlay polls this to draw the virtual cursor
|
|
194
209
|
/// Turn token that the user interrupted with Esc. Codex persists the same
|
|
195
210
|
/// fact as a marker file per conversation+turn; here the token travels with
|
|
@@ -206,6 +221,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
206
221
|
readonly Dictionary<string, ShotRegion> _shots = new Dictionary<string, ShotRegion>();
|
|
207
222
|
int _shotTtlMs = 120000;
|
|
208
223
|
int _shotSeq = 0;
|
|
224
|
+
readonly string _shotGeneration = Guid.NewGuid().ToString("N");
|
|
209
225
|
|
|
210
226
|
const uint GW_OWNER = 4;
|
|
211
227
|
const int GWL_STYLE = -16;
|
|
@@ -213,11 +229,16 @@ public class WindowsDriver : IPlatformDriver
|
|
|
213
229
|
|
|
214
230
|
public WindowsDriver()
|
|
215
231
|
{
|
|
232
|
+
Maintenance(); // startup repair: a previous session may have died with cursors hidden
|
|
216
233
|
try { SetThreadDpiAwarenessContext(new IntPtr(-4)); } catch { }
|
|
217
234
|
try { SetProcessDPIAware(); } catch { }
|
|
218
235
|
}
|
|
219
236
|
|
|
220
|
-
public void Dispose()
|
|
237
|
+
public void Dispose()
|
|
238
|
+
{
|
|
239
|
+
if (_cursorWatchdog != null) { _cursorWatchdog.Dispose(); _cursorWatchdog = null; }
|
|
240
|
+
StopOverlay();
|
|
241
|
+
}
|
|
221
242
|
|
|
222
243
|
// ---- approval plumbing ----
|
|
223
244
|
public Dictionary<string, object> Configure(Dictionary<string, object> p)
|
|
@@ -254,6 +275,13 @@ public class WindowsDriver : IPlatformDriver
|
|
|
254
275
|
return r;
|
|
255
276
|
}
|
|
256
277
|
|
|
278
|
+
public Dictionary<string, object> InvalidateState(Dictionary<string, object> p)
|
|
279
|
+
{
|
|
280
|
+
lock (_sync) { _snapshots.Clear(); _shots.Clear(); }
|
|
281
|
+
StopOverlay();
|
|
282
|
+
return new Dictionary<string, object> { { "invalidated", true } };
|
|
283
|
+
}
|
|
284
|
+
|
|
257
285
|
void AssertAppApproved(IntPtr h)
|
|
258
286
|
{
|
|
259
287
|
if (!_requireAppApproval) return;
|
|
@@ -340,18 +368,12 @@ public class WindowsDriver : IPlatformDriver
|
|
|
340
368
|
|
|
341
369
|
public Dictionary<string, object> OverlayShow(Dictionary<string, object> p)
|
|
342
370
|
{
|
|
343
|
-
string image = Json.GetString(p, "image");
|
|
344
|
-
if (!string.IsNullOrEmpty(image)) _overlayImage = image;
|
|
345
|
-
string crop = Json.GetString(p, "crop");
|
|
346
|
-
if (!string.IsNullOrEmpty(crop)) _overlayCrop = crop;
|
|
347
371
|
string text = Json.GetString(p, "text");
|
|
348
372
|
if (!string.IsNullOrEmpty(text)) _overlayText = text;
|
|
349
373
|
string hint = Json.GetString(p, "hint");
|
|
350
374
|
if (!string.IsNullOrEmpty(hint)) _overlayHint = hint;
|
|
351
375
|
string accent = Json.GetString(p, "accent");
|
|
352
376
|
if (!string.IsNullOrEmpty(accent)) _overlayAccent = accent;
|
|
353
|
-
string corner = Json.GetString(p, "corner");
|
|
354
|
-
if (!string.IsNullOrEmpty(corner)) _overlayCorner = corner;
|
|
355
377
|
EnsureOverlay(Json.GetLong(p, "ttlMs", 20000));
|
|
356
378
|
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
357
379
|
r["visible"] = true;
|
|
@@ -390,16 +412,24 @@ public class WindowsDriver : IPlatformDriver
|
|
|
390
412
|
args.Append(" --cancel-file \"").Append(_overlayCancelFile).Append('"');
|
|
391
413
|
args.Append(" --text \"").Append(_overlayText).Append('"');
|
|
392
414
|
args.Append(" --hint \"").Append(_overlayHint).Append('"');
|
|
415
|
+
args.Append(" --owner-pid ").Append(Process.GetCurrentProcess().Id);
|
|
393
416
|
args.Append(" --accent \"").Append(_overlayAccent).Append('"');
|
|
394
|
-
args.Append(" --corner ").Append(_overlayCorner);
|
|
395
417
|
if (_cursorFile != null) args.Append(" --cursor-file \"").Append(_cursorFile).Append('"');
|
|
396
|
-
|
|
418
|
+
// Suppression is the default (the virtual cursor stands in for the real one);
|
|
419
|
+
// DSH_CU_KEEP_CURSOR=1 keeps the real pointer visible instead.
|
|
420
|
+
if (Environment.GetEnvironmentVariable("DSH_CU_KEEP_CURSOR") == "1") args.Append(" --no-suppress-cursor");
|
|
397
421
|
try
|
|
398
422
|
{
|
|
399
423
|
ProcessStartInfo psi = new ProcessStartInfo(exe, args.ToString());
|
|
400
424
|
psi.UseShellExecute = false;
|
|
401
425
|
_overlay = Process.Start(psi);
|
|
402
426
|
_overlayUntil = DateTime.UtcNow.AddMilliseconds(ttlMs);
|
|
427
|
+
// Codex runs a dedicated system-cursor-manager process that restores the
|
|
428
|
+
// pointer the moment its parent dies. This timer gives the same guarantee
|
|
429
|
+
// without a second exe: if the overlay vanishes with cursors hidden, the
|
|
430
|
+
// pointer comes back within half a second, with no request needed.
|
|
431
|
+
if (_cursorWatchdog == null)
|
|
432
|
+
_cursorWatchdog = new System.Threading.Timer(delegate { try { Maintenance(); } catch { } }, null, 500, 500);
|
|
403
433
|
}
|
|
404
434
|
catch { _overlay = null; }
|
|
405
435
|
}
|
|
@@ -410,7 +440,8 @@ public class WindowsDriver : IPlatformDriver
|
|
|
410
440
|
// Kill() skips the overlay's exit handlers, which is where it would restore the
|
|
411
441
|
// system cursors. Restore them from here so a terminated overlay can never leave
|
|
412
442
|
// the user without a pointer.
|
|
413
|
-
|
|
443
|
+
RestoreCursors();
|
|
444
|
+
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
414
445
|
try { if (_overlayCancelFile != null && File.Exists(_overlayCancelFile)) File.Delete(_overlayCancelFile); } catch { }
|
|
415
446
|
_overlay = null;
|
|
416
447
|
_overlayUntil = DateTime.MinValue;
|
|
@@ -421,6 +452,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
421
452
|
|
|
422
453
|
void CheckCancel(Dictionary<string, object> p)
|
|
423
454
|
{
|
|
455
|
+
Budget.Check();
|
|
424
456
|
string token = p == null ? null : Json.GetString(p, "turnToken");
|
|
425
457
|
if (token != null && (token == _interruptedTurn || InterruptMarkerExists(token)))
|
|
426
458
|
throw new ToolError("turn_stopped", EscMessage);
|
|
@@ -435,7 +467,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
435
467
|
throw new ToolError("cancelled_by_user", EscMessage);
|
|
436
468
|
}
|
|
437
469
|
|
|
438
|
-
const int ProtocolVersion =
|
|
470
|
+
const int ProtocolVersion = 2;
|
|
439
471
|
|
|
440
472
|
static string InterruptMarkerPath(string token)
|
|
441
473
|
{
|
|
@@ -604,11 +636,50 @@ public class WindowsDriver : IPlatformDriver
|
|
|
604
636
|
{
|
|
605
637
|
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
606
638
|
d["platform"] = PlatformName;
|
|
639
|
+
d["pid"] = (long)Process.GetCurrentProcess().Id;
|
|
640
|
+
d["overlayPid"] = _overlay != null && !_overlay.HasExited ? (object)(long)_overlay.Id : null;
|
|
607
641
|
d["ok"] = true;
|
|
608
642
|
d["uia"] = true;
|
|
609
643
|
return d;
|
|
610
644
|
}
|
|
611
645
|
|
|
646
|
+
/// Run cheaply before every request: if a previous overlay died while it was hiding
|
|
647
|
+
/// the system cursors, put them back. A blank desktop cursor is worse than a stray
|
|
648
|
+
/// visible one, so this never assumes the overlay cleaned up after itself.
|
|
649
|
+
public void Maintenance()
|
|
650
|
+
{
|
|
651
|
+
try
|
|
652
|
+
{
|
|
653
|
+
string flag = OverlayCursorFlagPath();
|
|
654
|
+
if (!File.Exists(flag)) return;
|
|
655
|
+
int pid;
|
|
656
|
+
if (!int.TryParse(File.ReadAllText(flag).Trim(), out pid)) { RestoreCursors(); return; }
|
|
657
|
+
bool alive;
|
|
658
|
+
try { Process.GetProcessById(pid); alive = true; } catch { alive = false; }
|
|
659
|
+
if (!alive) { RestoreCursors(); try { File.Delete(flag); } catch { } }
|
|
660
|
+
}
|
|
661
|
+
catch { }
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
static string OverlayCursorFlagPath()
|
|
665
|
+
{
|
|
666
|
+
return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use", "cursor-hidden.flag");
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
static void RestoreCursors()
|
|
670
|
+
{
|
|
671
|
+
try { SystemParametersInfo(0x0057, 0, IntPtr.Zero, 0); } catch { }
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
public Dictionary<string, object> RestoreCursor(Dictionary<string, object> p)
|
|
675
|
+
{
|
|
676
|
+
RestoreCursors();
|
|
677
|
+
try { File.Delete(OverlayCursorFlagPath()); } catch { }
|
|
678
|
+
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
679
|
+
r["restored"] = true;
|
|
680
|
+
return r;
|
|
681
|
+
}
|
|
682
|
+
|
|
612
683
|
public Dictionary<string, object> Hello(Dictionary<string, object> p)
|
|
613
684
|
{
|
|
614
685
|
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
@@ -811,8 +882,13 @@ public class WindowsDriver : IPlatformDriver
|
|
|
811
882
|
for (int i = 0; i < args.Count; i++)
|
|
812
883
|
{
|
|
813
884
|
if (i > 0) argLine.Append(' ');
|
|
814
|
-
string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture);
|
|
815
|
-
|
|
885
|
+
string a = Convert.ToString(args[i], System.Globalization.CultureInfo.InvariantCulture) ?? "";
|
|
886
|
+
// A quote or newline would let a caller smuggle extra arguments past the
|
|
887
|
+
// single quoting rule below, so refuse instead of guessing.
|
|
888
|
+
if (a.IndexOf('"') >= 0 || a.IndexOf('\n') >= 0 || a.IndexOf('\r') >= 0)
|
|
889
|
+
throw new ToolError("bad_request", "launch_app argument " + i + " contains a quote or newline; pass a single plain argument");
|
|
890
|
+
if (a.Length == 0 || a.IndexOf(' ') >= 0 || a.IndexOf('\t') >= 0)
|
|
891
|
+
argLine.Append('"').Append(a).Append('"');
|
|
816
892
|
else argLine.Append(a);
|
|
817
893
|
}
|
|
818
894
|
psi.Arguments = argLine.ToString();
|
|
@@ -1096,7 +1172,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1096
1172
|
if (zIndex < 0) zIndex = total - 1;
|
|
1097
1173
|
Dictionary<string, object> shot = CaptureWindow(w);
|
|
1098
1174
|
shot["zIndex"] = (long)(total - zIndex); // larger = visually above
|
|
1099
|
-
string id = "shot-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
|
|
1175
|
+
string id = "shot-" + _shotGeneration + "-" + (++_shotSeq).ToString() + "-" + w.ToInt64().ToString();
|
|
1100
1176
|
shot["id"] = id;
|
|
1101
1177
|
RECT r = BoundsOf(w);
|
|
1102
1178
|
lock (_sync)
|
|
@@ -1405,19 +1481,20 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1405
1481
|
SendInput(1, arr, Marshal.SizeOf(typeof(INPUT)));
|
|
1406
1482
|
}
|
|
1407
1483
|
|
|
1408
|
-
void ClickAt(int screenX, int screenY, int count, string button)
|
|
1484
|
+
void ClickAt(int screenX, int screenY, int count, string button, Dictionary<string, object> p)
|
|
1409
1485
|
{
|
|
1410
1486
|
SetCursorPos(screenX, screenY);
|
|
1411
1487
|
System.Threading.Thread.Sleep(35);
|
|
1412
1488
|
for (int i = 0; i < Math.Max(1, count); i++)
|
|
1413
1489
|
{
|
|
1490
|
+
CheckCancel(p);
|
|
1414
1491
|
if (i > 0) System.Threading.Thread.Sleep(60);
|
|
1415
1492
|
uint down = MOUSEEVENTF_LEFTDOWN, up = MOUSEEVENTF_LEFTUP;
|
|
1416
1493
|
if (button == "r") { down = MOUSEEVENTF_RIGHTDOWN; up = MOUSEEVENTF_RIGHTUP; }
|
|
1417
1494
|
else if (button == "m") { down = MOUSEEVENTF_MIDDLEDOWN; up = MOUSEEVENTF_MIDDLEUP; }
|
|
1418
1495
|
SendMouse(down);
|
|
1419
|
-
System.Threading.Thread.Sleep(22);
|
|
1420
|
-
SendMouse(up);
|
|
1496
|
+
try { System.Threading.Thread.Sleep(22); }
|
|
1497
|
+
finally { SendMouse(up); }
|
|
1421
1498
|
}
|
|
1422
1499
|
}
|
|
1423
1500
|
|
|
@@ -1483,7 +1560,8 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1483
1560
|
|
|
1484
1561
|
Activate(h); // pointer clicks need a real foreground window
|
|
1485
1562
|
MarkCursor(sx, sy);
|
|
1486
|
-
|
|
1563
|
+
CheckCancel(p);
|
|
1564
|
+
ClickAt(sx, sy, count, button, p);
|
|
1487
1565
|
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1488
1566
|
r["clicked"] = "pointer";
|
|
1489
1567
|
r["screenX"] = (long)sx;
|
|
@@ -1511,7 +1589,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1511
1589
|
string key = Json.GetString(p, "key");
|
|
1512
1590
|
if (string.IsNullOrEmpty(key)) throw new ToolError("bad_request", "press_key requires 'key'");
|
|
1513
1591
|
Activate(h);
|
|
1514
|
-
Chord(key);
|
|
1592
|
+
Chord(key, p);
|
|
1515
1593
|
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1516
1594
|
r["pressed"] = key;
|
|
1517
1595
|
return r;
|
|
@@ -1569,7 +1647,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1569
1647
|
SendKey(vk, 0, flags);
|
|
1570
1648
|
}
|
|
1571
1649
|
|
|
1572
|
-
|
|
1650
|
+
void Chord(string chord, Dictionary<string, object> p)
|
|
1573
1651
|
{
|
|
1574
1652
|
string[] parts = chord.Split('+');
|
|
1575
1653
|
List<ushort> keys = new List<ushort>();
|
|
@@ -1580,8 +1658,17 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1580
1658
|
keys.Add(ResolveKey(token));
|
|
1581
1659
|
}
|
|
1582
1660
|
if (keys.Count == 0) throw new ToolError("bad_request", "empty key chord");
|
|
1583
|
-
|
|
1584
|
-
|
|
1661
|
+
int pressed = 0;
|
|
1662
|
+
try
|
|
1663
|
+
{
|
|
1664
|
+
for (int i = 0; i < keys.Count; i++)
|
|
1665
|
+
{
|
|
1666
|
+
CheckCancel(p);
|
|
1667
|
+
SendVk(keys[i], false); pressed++;
|
|
1668
|
+
System.Threading.Thread.Sleep(12);
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
finally { for (int i = pressed - 1; i >= 0; i--) SendVk(keys[i], true); }
|
|
1585
1672
|
}
|
|
1586
1673
|
|
|
1587
1674
|
/// The Windows/Meta key family is hard-denied: it opens Run, Settings and
|
|
@@ -1622,6 +1709,7 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1622
1709
|
Activate(h);
|
|
1623
1710
|
for (int i = 0; i < text.Length; i++)
|
|
1624
1711
|
{
|
|
1712
|
+
CheckCancel(p);
|
|
1625
1713
|
char c = text[i];
|
|
1626
1714
|
if (c == '\n') { SendVk(0x0D, false); SendVk(0x0D, true); continue; }
|
|
1627
1715
|
if (c == '\t') { SendVk(0x09, false); SendVk(0x09, true); continue; }
|
|
@@ -1642,13 +1730,16 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1642
1730
|
EnsureOverlay(8000);
|
|
1643
1731
|
IntPtr h = ResolveWindow(p);
|
|
1644
1732
|
AssertAppApproved(h);
|
|
1645
|
-
|
|
1733
|
+
// Validation first: an invalid call must not activate a window or move the pointer.
|
|
1646
1734
|
double scrollY = Json.GetDouble(p, "scrollY", 0);
|
|
1647
1735
|
double scrollX = Json.GetDouble(p, "scrollX", 0);
|
|
1648
1736
|
double x = Json.GetDouble(p, "x", double.NaN);
|
|
1649
1737
|
double y = Json.GetDouble(p, "y", double.NaN);
|
|
1650
1738
|
if (double.IsNaN(x) || double.IsNaN(y))
|
|
1651
1739
|
throw new ToolError("bad_request", "scroll requires finite 'x' and 'y' (the point to scroll from, in window/screenshot coordinates); click a pane first if it needs focus");
|
|
1740
|
+
if (Math.Abs(scrollX) < 0.0001 && Math.Abs(scrollY) < 0.0001)
|
|
1741
|
+
throw new ToolError("bad_request", "scroll requires a non-zero scrollX or scrollY");
|
|
1742
|
+
Activate(h);
|
|
1652
1743
|
int sx, sy;
|
|
1653
1744
|
string scrollShot = Json.GetString(p, "screenshotId");
|
|
1654
1745
|
int originLeft, originTop;
|
|
@@ -1685,6 +1776,12 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1685
1776
|
EnsureOverlay(8000);
|
|
1686
1777
|
IntPtr h = ResolveWindow(p);
|
|
1687
1778
|
AssertAppApproved(h);
|
|
1779
|
+
double fromX = Json.GetDouble(p, "from_x", double.NaN);
|
|
1780
|
+
double fromY = Json.GetDouble(p, "from_y", double.NaN);
|
|
1781
|
+
double toX = Json.GetDouble(p, "to_x", double.NaN);
|
|
1782
|
+
double toY = Json.GetDouble(p, "to_y", double.NaN);
|
|
1783
|
+
if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
|
|
1784
|
+
throw new ToolError("bad_request", "drag requires finite from_x/from_y/to_x/to_y (window-relative pixels)");
|
|
1688
1785
|
Activate(h);
|
|
1689
1786
|
int originX, originY;
|
|
1690
1787
|
double dragScaleX = 1.0, dragScaleY = 1.0;
|
|
@@ -1692,8 +1789,8 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1692
1789
|
if (!string.IsNullOrEmpty(dragShot))
|
|
1693
1790
|
{
|
|
1694
1791
|
ShotRegion region = ResolveShot(h, dragShot);
|
|
1695
|
-
AssertPointInsideShot(
|
|
1696
|
-
AssertPointInsideShot(
|
|
1792
|
+
AssertPointInsideShot(fromX, fromY, region, dragShot, "drag start");
|
|
1793
|
+
AssertPointInsideShot(toX, toY, region, dragShot, "drag end");
|
|
1697
1794
|
originX = region.OriginX; originY = region.OriginY;
|
|
1698
1795
|
dragScaleX = region.ScaleX; dragScaleY = region.ScaleY;
|
|
1699
1796
|
}
|
|
@@ -1702,26 +1799,32 @@ public class WindowsDriver : IPlatformDriver
|
|
|
1702
1799
|
RECT b = BoundsOf(h);
|
|
1703
1800
|
originX = b.Left; originY = b.Top;
|
|
1704
1801
|
}
|
|
1705
|
-
int fx = originX + (int)Math.Round(
|
|
1706
|
-
int fy = originY + (int)Math.Round(
|
|
1707
|
-
int tx = originX + (int)Math.Round(
|
|
1708
|
-
int ty = originY + (int)Math.Round(
|
|
1802
|
+
int fx = originX + (int)Math.Round(fromX * dragScaleX);
|
|
1803
|
+
int fy = originY + (int)Math.Round(fromY * dragScaleY);
|
|
1804
|
+
int tx = originX + (int)Math.Round(toX * dragScaleX);
|
|
1805
|
+
int ty = originY + (int)Math.Round(toY * dragScaleY);
|
|
1709
1806
|
|
|
1710
1807
|
MarkCursor(fx, fy);
|
|
1711
1808
|
SetCursorPos(fx, fy);
|
|
1712
1809
|
System.Threading.Thread.Sleep(60);
|
|
1810
|
+
CheckCancel(p);
|
|
1713
1811
|
SendMouse(MOUSEEVENTF_LEFTDOWN);
|
|
1714
|
-
|
|
1715
|
-
const int steps = 14;
|
|
1716
|
-
for (int i = 1; i <= steps; i++)
|
|
1812
|
+
try
|
|
1717
1813
|
{
|
|
1718
|
-
|
|
1719
|
-
int
|
|
1720
|
-
|
|
1721
|
-
|
|
1814
|
+
System.Threading.Thread.Sleep(80);
|
|
1815
|
+
const int steps = 14;
|
|
1816
|
+
for (int i = 1; i <= steps; i++)
|
|
1817
|
+
{
|
|
1818
|
+
CheckCancel(p);
|
|
1819
|
+
int x = fx + (tx - fx) * i / steps;
|
|
1820
|
+
int y = fy + (ty - fy) * i / steps;
|
|
1821
|
+
SetCursorPos(x, y);
|
|
1822
|
+
System.Threading.Thread.Sleep(18);
|
|
1823
|
+
}
|
|
1824
|
+
System.Threading.Thread.Sleep(90);
|
|
1825
|
+
CheckCancel(p);
|
|
1722
1826
|
}
|
|
1723
|
-
|
|
1724
|
-
SendMouse(MOUSEEVENTF_LEFTUP);
|
|
1827
|
+
finally { SendMouse(MOUSEEVENTF_LEFTUP); }
|
|
1725
1828
|
|
|
1726
1829
|
Dictionary<string, object> r = new Dictionary<string, object>();
|
|
1727
1830
|
r["dragged"] = true;
|
package/native/Overlay.cs
CHANGED
|
@@ -22,6 +22,7 @@ public sealed class Config
|
|
|
22
22
|
public static string Title = "DSH 正在使用您的电脑";
|
|
23
23
|
public static string Hint = "Esc 取消";
|
|
24
24
|
public static bool SuppressCursor = false;
|
|
25
|
+
public static int OwnerPid = 0;
|
|
25
26
|
public static int Corner1 = 0; // unused placeholder kept for arg compatibility
|
|
26
27
|
public static Color Accent = Color.FromArgb(63, 127, 216);
|
|
27
28
|
}
|
|
@@ -52,9 +53,6 @@ public static class Native
|
|
|
52
53
|
public const int ULW_ALPHA = 2;
|
|
53
54
|
public const uint SPI_SETCURSORS = 0x0057;
|
|
54
55
|
|
|
55
|
-
public static bool LastLayerOk = true;
|
|
56
|
-
public static int LastLayerError = 0;
|
|
57
|
-
|
|
58
56
|
public static void ApplyLayered(Form form, Bitmap surface)
|
|
59
57
|
{
|
|
60
58
|
IntPtr screenDc = GetDC(IntPtr.Zero);
|
|
@@ -67,8 +65,7 @@ public static class Native
|
|
|
67
65
|
POINT src = new POINT { X = 0, Y = 0 };
|
|
68
66
|
SIZE size = new SIZE { CX = surface.Width, CY = surface.Height };
|
|
69
67
|
BLENDFUNCTION blend = new BLENDFUNCTION { BlendOp = 0, BlendFlags = 0, SourceConstantAlpha = 255, AlphaFormat = 1 };
|
|
70
|
-
|
|
71
|
-
LastLayerError = LastLayerOk ? 0 : Marshal.GetLastWin32Error();
|
|
68
|
+
UpdateLayeredWindow(form.Handle, screenDc, ref dst, ref size, memDc, ref src, 0, ref blend, ULW_ALPHA);
|
|
72
69
|
}
|
|
73
70
|
finally
|
|
74
71
|
{
|
|
@@ -84,19 +81,48 @@ public static class Native
|
|
|
84
81
|
try { SetWindowPos(form.Handle, HWND_TOPMOST, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW); } catch { }
|
|
85
82
|
}
|
|
86
83
|
|
|
84
|
+
/// Path of the "cursors are hidden" flag. It carries our PID so a long-lived
|
|
85
|
+
/// supervisor can tell "the overlay is still hiding them" from "the overlay died
|
|
86
|
+
/// and left the desktop without a pointer".
|
|
87
|
+
public static string CursorFlagPath()
|
|
88
|
+
{
|
|
89
|
+
string dir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "dsh-computer-use");
|
|
90
|
+
return Path.Combine(dir, "cursor-hidden.flag");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
static IntPtr _blankCursor;
|
|
94
|
+
|
|
87
95
|
public static void SuppressSystemCursors(bool suppress)
|
|
88
96
|
{
|
|
89
97
|
try
|
|
90
98
|
{
|
|
91
|
-
if (!suppress)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
if (!suppress)
|
|
100
|
+
{
|
|
101
|
+
SystemParametersInfo(SPI_SETCURSORS, 0, IntPtr.Zero, 0);
|
|
102
|
+
try { File.Delete(CursorFlagPath()); } catch { }
|
|
103
|
+
if (_blankCursor != IntPtr.Zero) { try { DestroyCursor(_blankCursor); } catch { } _blankCursor = IntPtr.Zero; }
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
// Cache the blank cursor: Windows resets the cursors on resume, display
|
|
107
|
+
// change and some logins, so the controller calls this again periodically
|
|
108
|
+
// (Codex calls the same "system cursor re-suppression").
|
|
109
|
+
if (_blankCursor == IntPtr.Zero)
|
|
110
|
+
{
|
|
111
|
+
byte[] andMask = new byte[128];
|
|
112
|
+
byte[] xorMask = new byte[128];
|
|
113
|
+
for (int i = 0; i < andMask.Length; i++) andMask[i] = 0xFF;
|
|
114
|
+
_blankCursor = CreateCursor(IntPtr.Zero, 0, 0, 32, 32, andMask, xorMask);
|
|
115
|
+
}
|
|
116
|
+
IntPtr blank = _blankCursor;
|
|
96
117
|
if (blank == IntPtr.Zero) return;
|
|
97
118
|
uint[] ids = new uint[] { 32512, 32513, 32514, 32515, 32516, 32631, 32640, 32641, 32642, 32643, 32644, 32645, 32646, 32648, 32649, 32650, 32651, 32671 };
|
|
98
119
|
foreach (uint id in ids) SetSystemCursor(CopyIcon(blank), id);
|
|
99
|
-
|
|
120
|
+
try
|
|
121
|
+
{
|
|
122
|
+
Directory.CreateDirectory(Path.GetDirectoryName(CursorFlagPath()));
|
|
123
|
+
File.WriteAllText(CursorFlagPath(), System.Diagnostics.Process.GetCurrentProcess().Id.ToString());
|
|
124
|
+
}
|
|
125
|
+
catch { }
|
|
100
126
|
}
|
|
101
127
|
catch { }
|
|
102
128
|
}
|
|
@@ -204,19 +230,6 @@ public sealed class CursorForm : Form
|
|
|
204
230
|
g.DrawPolygon(outline, arrow);
|
|
205
231
|
}
|
|
206
232
|
}
|
|
207
|
-
string dump = Environment.GetEnvironmentVariable("DSH_CU_DUMP");
|
|
208
|
-
if (!string.IsNullOrEmpty(dump))
|
|
209
|
-
{
|
|
210
|
-
try
|
|
211
|
-
{
|
|
212
|
-
surface.Save(dump, ImageFormat.Png);
|
|
213
|
-
System.IO.File.AppendAllText(dump + ".log",
|
|
214
|
-
DateTime.Now.ToString("HH:mm:ss.fff") + " visible=" + Visible + " handle=" + Handle.ToInt64() +
|
|
215
|
-
" bounds=" + Left + "," + Top + " " + Width + "x" + Height +
|
|
216
|
-
" layerOk=" + Native.LastLayerOk + " err=" + Native.LastLayerError + "\n");
|
|
217
|
-
}
|
|
218
|
-
catch { }
|
|
219
|
-
}
|
|
220
233
|
if (!Visible) Show();
|
|
221
234
|
Native.ApplyLayered(this, surface);
|
|
222
235
|
}
|
|
@@ -391,7 +404,7 @@ public sealed class StripForm : Form
|
|
|
391
404
|
// halo: widest + faintest first, then tighten
|
|
392
405
|
for (int i = 3; i >= 1; i--)
|
|
393
406
|
{
|
|
394
|
-
float width = i *
|
|
407
|
+
float width = i * 3.2f;
|
|
395
408
|
int alpha = Math.Max(6, glowAlpha / (i + 1));
|
|
396
409
|
using (Pen halo = new Pen(Color.FromArgb(alpha, accent), width))
|
|
397
410
|
{
|
|
@@ -410,7 +423,7 @@ public sealed class StripForm : Form
|
|
|
410
423
|
}
|
|
411
424
|
|
|
412
425
|
// core hairline
|
|
413
|
-
using (Pen core = new Pen(Color.FromArgb(coreAlpha, accent),
|
|
426
|
+
using (Pen core = new Pen(Color.FromArgb(coreAlpha, accent), 2f))
|
|
414
427
|
{
|
|
415
428
|
core.Alignment = PenAlignment.Center;
|
|
416
429
|
if (vertical)
|
|
@@ -514,7 +527,7 @@ internal sealed class Controller : ApplicationContext
|
|
|
514
527
|
delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
515
528
|
|
|
516
529
|
const int Radius = 18; // corner radius
|
|
517
|
-
const int Glow =
|
|
530
|
+
const int Glow = 16; // band the halo fades inside
|
|
518
531
|
|
|
519
532
|
public Controller(int ttlMs)
|
|
520
533
|
{
|
|
@@ -545,13 +558,18 @@ internal sealed class Controller : ApplicationContext
|
|
|
545
558
|
void Tick()
|
|
546
559
|
{
|
|
547
560
|
_tick++;
|
|
561
|
+
if (Config.OwnerPid > 0)
|
|
562
|
+
{
|
|
563
|
+
try { if (System.Diagnostics.Process.GetProcessById(Config.OwnerPid).HasExited) { ExitThreadCore(); return; } }
|
|
564
|
+
catch { ExitThreadCore(); return; }
|
|
565
|
+
}
|
|
548
566
|
if (DateTime.UtcNow > _deadline) { ExitThreadCore(); return; }
|
|
549
567
|
|
|
550
568
|
Rectangle screen = SystemInformation.VirtualScreen;
|
|
551
569
|
// slow 3.2s breath: the frame gets a little brighter and a touch wider
|
|
552
570
|
double phase = (Math.Sin(_tick / 26.0 - 1.57) + 1.0) / 2.0;
|
|
553
|
-
int coreAlpha =
|
|
554
|
-
int glowAlpha =
|
|
571
|
+
int coreAlpha = 200 + (int)Math.Round(55 * phase); // clearly visible, still a hairline
|
|
572
|
+
int glowAlpha = 42 + (int)Math.Round(26 * phase);
|
|
555
573
|
int thickness = Glow;
|
|
556
574
|
|
|
557
575
|
int x0 = screen.Left, y0 = screen.Top, w = screen.Width, h = screen.Height;
|
|
@@ -602,6 +620,10 @@ internal sealed class Controller : ApplicationContext
|
|
|
602
620
|
}
|
|
603
621
|
catch { }
|
|
604
622
|
|
|
623
|
+
// Windows restores the real cursors on resume / display change; put the blank
|
|
624
|
+
// ones back while the indicator is up (Codex: "schedule system cursor re-suppression").
|
|
625
|
+
if (Config.SuppressCursor && _tick % 40 == 0) Native.SuppressSystemCursors(true);
|
|
626
|
+
|
|
605
627
|
if (!_cursor.Live)
|
|
606
628
|
{
|
|
607
629
|
PointNative p;
|
|
@@ -641,6 +663,36 @@ internal static class Program
|
|
|
641
663
|
try { SetProcessDpiAwarenessContext(new IntPtr(-4)); } catch { }
|
|
642
664
|
try { SetProcessDPIAware(); } catch { }
|
|
643
665
|
|
|
666
|
+
// Guard mode: wait for the owner, then restore the cursors. This must run before
|
|
667
|
+
// anything else - a guard that reached the suppression block would hide the
|
|
668
|
+
// cursors again and spawn another guard, and the chain would never stop.
|
|
669
|
+
int guardIndex = Array.IndexOf(args, "--cursor-guard");
|
|
670
|
+
if (guardIndex >= 0 && args.Length > guardIndex + 1)
|
|
671
|
+
{
|
|
672
|
+
int ownerPid;
|
|
673
|
+
if (int.TryParse(args[guardIndex + 1], out ownerPid) && ownerPid != System.Diagnostics.Process.GetCurrentProcess().Id)
|
|
674
|
+
{
|
|
675
|
+
try
|
|
676
|
+
{
|
|
677
|
+
System.Diagnostics.Process owner = System.Diagnostics.Process.GetProcessById(ownerPid);
|
|
678
|
+
owner.WaitForExit(600000); // hard cap: a guard never lives forever
|
|
679
|
+
}
|
|
680
|
+
catch { /* owner already gone */ }
|
|
681
|
+
// Only clean up our own owner's suppression: a newer overlay may have
|
|
682
|
+
// taken over the cursors in the meantime, and restoring them here would
|
|
683
|
+
// make the real pointer reappear under a live indicator.
|
|
684
|
+
bool ours = false;
|
|
685
|
+
try
|
|
686
|
+
{
|
|
687
|
+
string flagPath = Native.CursorFlagPath();
|
|
688
|
+
ours = File.Exists(flagPath) && File.ReadAllText(flagPath).Trim() == ownerPid.ToString();
|
|
689
|
+
}
|
|
690
|
+
catch { }
|
|
691
|
+
if (ours) Native.SuppressSystemCursors(false);
|
|
692
|
+
}
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
|
|
644
696
|
int ttlMs = 20000;
|
|
645
697
|
for (int i = 0; i < args.Length - 1; i++)
|
|
646
698
|
{
|
|
@@ -651,6 +703,7 @@ internal static class Program
|
|
|
651
703
|
else if (a == "--text") Config.Title = args[i + 1];
|
|
652
704
|
else if (a == "--hint") Config.Hint = args[i + 1];
|
|
653
705
|
else if (a == "--ttl-ms") int.TryParse(args[i + 1], out ttlMs);
|
|
706
|
+
else if (a == "--owner-pid") int.TryParse(args[i + 1], out Config.OwnerPid);
|
|
654
707
|
}
|
|
655
708
|
Config.SuppressCursor = true; // Codex default
|
|
656
709
|
for (int i = 0; i < args.Length; i++) if (args[i] == "--no-suppress-cursor") Config.SuppressCursor = false;
|
|
@@ -660,6 +713,16 @@ internal static class Program
|
|
|
660
713
|
{
|
|
661
714
|
Native.SuppressSystemCursors(true);
|
|
662
715
|
AppDomain.CurrentDomain.ProcessExit += (s, e) => Native.SuppressSystemCursors(false);
|
|
716
|
+
try
|
|
717
|
+
{
|
|
718
|
+
System.Diagnostics.ProcessStartInfo guard = new System.Diagnostics.ProcessStartInfo(
|
|
719
|
+
System.Reflection.Assembly.GetExecutingAssembly().Location,
|
|
720
|
+
"--cursor-guard " + System.Diagnostics.Process.GetCurrentProcess().Id);
|
|
721
|
+
guard.UseShellExecute = false;
|
|
722
|
+
guard.CreateNoWindow = true;
|
|
723
|
+
System.Diagnostics.Process.Start(guard);
|
|
724
|
+
}
|
|
725
|
+
catch { /* the helper watchdog still covers us */ }
|
|
663
726
|
}
|
|
664
727
|
|
|
665
728
|
Application.EnableVisualStyles();
|
package/native/Program.cs
CHANGED
|
@@ -88,16 +88,19 @@ public static class Program
|
|
|
88
88
|
if (p == null) p = new Dictionary<string, object>();
|
|
89
89
|
// Codex ships the budget with every call (x-oai-cua-request-budget-ms) and the
|
|
90
90
|
// helper honours it; here it bounds capture waits and marks overruns.
|
|
91
|
-
Budget.Begin(Json.GetLong(p, "__budgetMs", 0));
|
|
91
|
+
Budget.Begin(Json.GetLong(p, "__budgetMs", 0), Json.GetString(p, "__cancelFile"));
|
|
92
|
+
try { if (_driver != null) _driver.Maintenance(); } catch { }
|
|
92
93
|
object result = Dispatch(method, p);
|
|
93
94
|
return Ok(id, result);
|
|
94
95
|
}
|
|
95
96
|
catch (ToolError te)
|
|
96
97
|
{
|
|
98
|
+
try { _driver.InvalidateState(new Dictionary<string, object>()); } catch { }
|
|
97
99
|
return Fail(id, te.Code, te.Message);
|
|
98
100
|
}
|
|
99
101
|
catch (Exception ex)
|
|
100
102
|
{
|
|
103
|
+
try { _driver.InvalidateState(new Dictionary<string, object>()); } catch { }
|
|
101
104
|
return Fail(id, "internal_error", ex.GetType().Name + ": " + ex.Message);
|
|
102
105
|
}
|
|
103
106
|
}
|
|
@@ -109,6 +112,7 @@ public static class Program
|
|
|
109
112
|
{
|
|
110
113
|
case "ping": return _driver.Ping(p);
|
|
111
114
|
case "hello": return _driver.Hello(p);
|
|
115
|
+
case "restore_cursor": return _driver.RestoreCursor(p);
|
|
112
116
|
case "close": return _driver.Close(p);
|
|
113
117
|
case "list_windows": return _driver.ListWindows(p);
|
|
114
118
|
case "list_apps": return _driver.ListApps(p);
|
|
@@ -128,6 +132,7 @@ public static class Program
|
|
|
128
132
|
case "configure": return _driver.Configure(p);
|
|
129
133
|
case "approve_app": return _driver.ApproveApp(p);
|
|
130
134
|
case "end_turn": return _driver.EndTurn(p);
|
|
135
|
+
case "invalidate_state": return _driver.InvalidateState(p);
|
|
131
136
|
case "start_audio_recording": return _driver.StartAudioRecording(p);
|
|
132
137
|
case "stop_audio_recording": return _driver.StopAudioRecording(p);
|
|
133
138
|
case "shutdown": return new Dictionary<string, object> { { "bye", true } };
|