dsh-codex-style-computer-use 1.3.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ar.md +234 -0
- package/README.de.md +234 -0
- package/README.es.md +264 -0
- package/README.fa.md +234 -0
- package/README.fr.md +243 -0
- package/README.md +72 -17
- package/README.pt.md +234 -0
- package/README.ru.md +259 -0
- package/README.zh.md +66 -16
- package/icon.svg +20 -4
- package/lib/audit.js +7 -0
- package/lib/batch.js +14 -19
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/compact-feedback.js +175 -0
- package/lib/continuation.js +1 -1
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/index.js +403 -136
- package/lib/core/native-runtime.js +17 -0
- package/lib/core/worker.cs +2160 -229
- package/lib/feedback-routes.json +11 -0
- package/lib/feedback.js +49 -16
- package/lib/image-output.js +87 -17
- package/lib/index.js +25 -7
- package/lib/key-command.js +37 -0
- package/lib/launch.js +67 -0
- package/lib/media-contract.js +43 -0
- package/lib/observations.js +267 -13
- package/lib/policy.js +29 -9
- package/lib/tool-contract.js +56 -0
- package/lib/tools.js +203 -52
- package/lib/verification-region.js +17 -0
- package/mcp/package.json +2 -1
- package/mcp/src/execution.js +8 -5
- package/mcp/src/index.js +16 -267
- package/mcp/src/tools.js +271 -0
- package/package.json +6 -1
- package/scripts/check-behavior.mjs +83 -0
- package/scripts/measure-tool-tokens.mjs +21 -0
- package/scripts/verify-package.mjs +9 -0
- package/skills/computer-use/SKILL.md +55 -43
- package/skills/computer-use/references/lifecycle.md +11 -7
- package/skills/computer-use/references/maintenance.md +9 -5
- package/skills/computer-use/references/precision.md +39 -14
- package/skills/computer-use/references/progressive-control.md +10 -4
package/lib/core/worker.cs
CHANGED
|
@@ -20,6 +20,7 @@ using System.Collections.Generic;
|
|
|
20
20
|
using System.Collections.Concurrent;
|
|
21
21
|
using System.Diagnostics;
|
|
22
22
|
using System.Drawing;
|
|
23
|
+
using System.Drawing.Drawing2D;
|
|
23
24
|
using System.Drawing.Imaging;
|
|
24
25
|
using System.IO;
|
|
25
26
|
using System.Linq;
|
|
@@ -36,19 +37,34 @@ using Windows.Graphics.Imaging;
|
|
|
36
37
|
|
|
37
38
|
namespace DshComputerUse.Worker
|
|
38
39
|
{
|
|
39
|
-
//
|
|
40
|
+
// On-demand cadence: one monotonic timestamp, no timer, history list or disk writes.
|
|
41
|
+
// Only a successfully displayed cue advances the sliding three-minute window.
|
|
42
|
+
public sealed class PreparationCadence
|
|
43
|
+
{
|
|
44
|
+
public const int FullMs = 1000, RepeatMs = 100, RepeatWindowMs = 180000;
|
|
45
|
+
long lastShown = -1;
|
|
46
|
+
public int Duration(long now)
|
|
47
|
+
{ long previous = Interlocked.Read(ref lastShown); return previous >= 0 && now >= previous && now - previous < RepeatWindowMs ? RepeatMs : FullMs; }
|
|
48
|
+
public void Presented(long now) { Interlocked.Exchange(ref lastShown, now); }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// A live cycle observes physical holds; actuation or an explicit scene hold can start a handoff.
|
|
40
52
|
// Once yielded, three quiet seconds with no held input returns control to the agent.
|
|
41
53
|
public sealed class HumanInputState
|
|
42
54
|
{
|
|
43
55
|
public const int SustainedMs = 2000, IdleMs = 3000;
|
|
44
56
|
readonly object sync = new object();
|
|
45
57
|
readonly HashSet<int> held = new HashSet<int>();
|
|
58
|
+
readonly Dictionary<int, int> releasedSamples = new Dictionary<int, int>();
|
|
59
|
+
readonly HashSet<int> injectedTouched = new HashSet<int>();
|
|
46
60
|
bool enabled, acting, seen;
|
|
47
61
|
string phase = "idle", lastKind = "none";
|
|
48
|
-
long first, last, revision, burst;
|
|
62
|
+
long first, last, revision, burst, stateSeq;
|
|
63
|
+
int reconciledReleases;
|
|
49
64
|
public string Phase { get { lock (sync) { return phase; } } }
|
|
50
65
|
public long Revision { get { lock (sync) { return revision; } } }
|
|
51
66
|
public bool Active { get { lock (sync) { return phase != "idle"; } } }
|
|
67
|
+
public bool HasHeld { get { lock (sync) { return held.Count != 0; } } }
|
|
52
68
|
public string LastKind { get { lock (sync) { return lastKind; } } }
|
|
53
69
|
public bool Held(int vk)
|
|
54
70
|
{
|
|
@@ -69,7 +85,7 @@ namespace DshComputerUse.Worker
|
|
|
69
85
|
lock (sync)
|
|
70
86
|
{
|
|
71
87
|
if (enabled == on) return;
|
|
72
|
-
enabled = on;
|
|
88
|
+
enabled = on; stateSeq++;
|
|
73
89
|
if (!on) { phase = "idle"; acting = false; }
|
|
74
90
|
}
|
|
75
91
|
}
|
|
@@ -77,30 +93,30 @@ namespace DshComputerUse.Worker
|
|
|
77
93
|
{
|
|
78
94
|
lock (sync)
|
|
79
95
|
{
|
|
80
|
-
acting = on;
|
|
96
|
+
if (acting != on) { acting = on; stateSeq++; }
|
|
81
97
|
// Do not replay earlier ordinary activity. A key/button still physically
|
|
82
98
|
// held does need ownership before we can inject a conflicting chord.
|
|
83
99
|
if (on && enabled && phase == "idle" && held.Count > 0)
|
|
84
|
-
{ last = now; revision++; Start(now); }
|
|
100
|
+
{ last = now; revision++; stateSeq++; Start(now); }
|
|
85
101
|
}
|
|
86
102
|
}
|
|
87
103
|
void Start(long now) { phase = "yielding"; first = now; if (!seen) last = now; burst++; }
|
|
88
104
|
public void Record(int key, bool down, bool up, long now) { Record(key, down, up, now, false); }
|
|
89
|
-
///
|
|
90
|
-
///
|
|
91
|
-
/// 2026-11-04: while a cycle is live, the human reaching for the mouse is a takeover no
|
|
92
|
-
/// matter whether an injection happens to be in flight at that instant — the old gate
|
|
93
|
-
/// (`idle && !acting`) only lit the yield during the few dozen milliseconds of each
|
|
94
|
-
/// click, so an interruption in the gaps showed no light at all.
|
|
105
|
+
/// Pointer jitter is filtered before this call. The protection flag includes both
|
|
106
|
+
/// actual injection and a retained scene hold; free observation never takes ownership.
|
|
95
107
|
public void Record(int key, bool down, bool up, long now, bool deliberate)
|
|
96
108
|
{
|
|
97
109
|
lock (sync)
|
|
98
110
|
{
|
|
99
111
|
bool continued = held.Count > 0 || (seen && now - last < IdleMs);
|
|
112
|
+
stateSeq++;
|
|
100
113
|
lastKind = key == 0 ? "pointer-or-wheel" : key <= 6 ? "button" : "keyboard";
|
|
101
|
-
if (key != 0) {
|
|
114
|
+
if (key != 0) {
|
|
115
|
+
if (down) held.Add(key); if (up) held.Remove(key);
|
|
116
|
+
releasedSamples.Remove(key); injectedTouched.Remove(key);
|
|
117
|
+
}
|
|
102
118
|
seen = true; last = now;
|
|
103
|
-
if (!enabled || (phase == "idle" && !acting
|
|
119
|
+
if (!enabled || (phase == "idle" && !acting)) return;
|
|
104
120
|
revision++;
|
|
105
121
|
if (phase == "idle" || !continued) Start(now);
|
|
106
122
|
if (now - first >= SustainedMs) phase = "waiting";
|
|
@@ -111,19 +127,77 @@ namespace DshComputerUse.Worker
|
|
|
111
127
|
lock (sync)
|
|
112
128
|
{
|
|
113
129
|
if (!enabled || phase == "idle") return;
|
|
114
|
-
if (held.Count == 0 && now - last >= IdleMs) { phase = "idle"; return; }
|
|
115
|
-
if (held.Count > 0 && now - first >= SustainedMs) phase = "waiting";
|
|
130
|
+
if (held.Count == 0 && now - last >= IdleMs) { phase = "idle"; stateSeq++; return; }
|
|
131
|
+
if (held.Count > 0 && now - first >= SustainedMs && phase != "waiting") { phase = "waiting"; stateSeq++; }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
public void NoteInjected(int key)
|
|
135
|
+
{
|
|
136
|
+
// An injected UP can make async state look released while a human
|
|
137
|
+
// still physically holds the same key. Never reconcile that case.
|
|
138
|
+
lock (sync) {
|
|
139
|
+
foreach (int tracked in held)
|
|
140
|
+
if (tracked == key || ModifierFamily(tracked) != 0 && ModifierFamily(tracked) == ModifierFamily(key) ||
|
|
141
|
+
(tracked == 1 || tracked == 2) && (key == 1 || key == 2))
|
|
142
|
+
{ injectedTouched.Add(tracked); releasedSamples.Remove(tracked); }
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
static int ModifierFamily(int key)
|
|
146
|
+
{ return key == 0x10 || key == 0xA0 || key == 0xA1 ? 0x10 : key == 0x11 || key == 0xA2 || key == 0xA3 ? 0x11 : key == 0x12 || key == 0xA4 || key == 0xA5 ? 0x12 : 0; }
|
|
147
|
+
public static bool AsyncButtonMayBeDown(int key, bool swapped, Func<int, bool> isDown)
|
|
148
|
+
{
|
|
149
|
+
// Async state names PHYSICAL buttons. With swapped primary/secondary
|
|
150
|
+
// semantics, require BOTH physical buttons up before repairing either
|
|
151
|
+
// history entry; never infer the low-level hook's mapping from a UI label.
|
|
152
|
+
return swapped && (key == 1 || key == 2) ? isDown(1) || isDown(2) : isDown(key);
|
|
153
|
+
}
|
|
154
|
+
public int ReconcileReleased(Func<int, bool> isDown, bool trusted, long now)
|
|
155
|
+
{
|
|
156
|
+
lock (sync) {
|
|
157
|
+
if (!trusted) { releasedSamples.Clear(); return 0; }
|
|
158
|
+
int removed = 0;
|
|
159
|
+
foreach (int key in held.ToArray()) {
|
|
160
|
+
if (injectedTouched.Contains(key) || isDown(key)) { releasedSamples.Remove(key); continue; }
|
|
161
|
+
int count; releasedSamples.TryGetValue(key, out count);
|
|
162
|
+
if (++count < 3) { releasedSamples[key] = count; continue; }
|
|
163
|
+
held.Remove(key); releasedSamples.Remove(key); removed++;
|
|
164
|
+
}
|
|
165
|
+
if (removed > 0) {
|
|
166
|
+
// This is fresh release evidence, not permission from elapsed
|
|
167
|
+
// time: still require the full quiet interval after this point.
|
|
168
|
+
last = now; seen = true; lastKind = "reconciled-release";
|
|
169
|
+
reconciledReleases += removed; revision++; stateSeq++;
|
|
170
|
+
}
|
|
171
|
+
return removed;
|
|
116
172
|
}
|
|
117
173
|
}
|
|
118
174
|
public object Snapshot(long now)
|
|
119
175
|
{
|
|
120
176
|
lock (sync) return Program.Dict("phase", phase, "active", phase != "idle",
|
|
121
|
-
"revision", revision, "burst", burst, "heldCount", held.Count, "lastKind", lastKind,
|
|
177
|
+
"revision", revision, "stateSeq", stateSeq, "burst", burst, "heldCount", held.Count, "lastKind", lastKind,
|
|
178
|
+
"reconciledReleaseCount", reconciledReleases,
|
|
122
179
|
"quietRemainingMs", phase == "idle" ? 0 : held.Count > 0 ? IdleMs : Math.Max(0, IdleMs - (now - last)),
|
|
123
180
|
"sustainedMs", SustainedMs, "idleMs", IdleMs);
|
|
124
181
|
}
|
|
125
182
|
}
|
|
126
183
|
|
|
184
|
+
// Domain ownership survives renderer failures and visibility changes. Glow
|
|
185
|
+
// only reads this state; cycle/operation lifecycle explicitly changes it.
|
|
186
|
+
public static class SceneControl
|
|
187
|
+
{
|
|
188
|
+
static volatile bool acting, holding;
|
|
189
|
+
public static bool Acting { get { return acting; } }
|
|
190
|
+
public static bool Holding { get { return holding; } }
|
|
191
|
+
public static void SetHolding(bool on)
|
|
192
|
+
{
|
|
193
|
+
holding = on && Panic.CycleLit && !Panic.Exited && !Panic.Stopped && !Panic.Engaged;
|
|
194
|
+
Panic.UpdateInputProtection();
|
|
195
|
+
}
|
|
196
|
+
public static void BeginActuation() { acting = true; SetHolding(true); }
|
|
197
|
+
public static void EndActuation() { acting = false; Panic.UpdateInputProtection(); }
|
|
198
|
+
public static void Reset() { acting = holding = false; Panic.UpdateInputProtection(); }
|
|
199
|
+
}
|
|
200
|
+
|
|
127
201
|
// The UAC secure desktop (and the lock screen) is the ONE surface automation must never
|
|
128
202
|
// reach — Windows enforces that itself. The honest addition (human request 2026-11-04)
|
|
129
203
|
// is to NOTICE the switch, refuse to dispatch while it stands, tell the human exactly
|
|
@@ -415,8 +489,22 @@ namespace DshComputerUse.Worker
|
|
|
415
489
|
// ---- focus / integrity probing (2026-09-13 "typing never lands" investigation) ----
|
|
416
490
|
[DllImport("user32.dll")]
|
|
417
491
|
public static extern IntPtr GetParent(IntPtr hWnd);
|
|
492
|
+
// GW_OWNER=4: a modal dialog's owner is the window that opened it (SameInputFamily).
|
|
418
493
|
[DllImport("user32.dll")]
|
|
419
|
-
public static extern
|
|
494
|
+
public static extern IntPtr GetWindow(IntPtr hWnd, uint cmd);
|
|
495
|
+
// ---- caret evidence: where WOULD the next character go? ----
|
|
496
|
+
// GUITHREADINFO reports the caret for the FOREGROUND THREAD without AttachThreadInput, and
|
|
497
|
+
// works in apps where UIA has no TextPattern (Word's document, many Win32 editors).
|
|
498
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
499
|
+
public struct GUITHREADINFO
|
|
500
|
+
{
|
|
501
|
+
public int cbSize;
|
|
502
|
+
public int flags;
|
|
503
|
+
public IntPtr hwndActive, hwndFocus, hwndCapture, hwndMenuOwner, hwndMoveSize, hwndCaret;
|
|
504
|
+
public RECT rcCaret;
|
|
505
|
+
}
|
|
506
|
+
[DllImport("user32.dll")] public static extern bool GetGUIThreadInfo(uint idThread, ref GUITHREADINFO info);
|
|
507
|
+
[DllImport("user32.dll")] public static extern bool IsWindow(IntPtr hWnd);
|
|
420
508
|
[DllImport("user32.dll")]
|
|
421
509
|
public static extern IntPtr WindowFromPoint(POINT p);
|
|
422
510
|
[DllImport("user32.dll")]
|
|
@@ -438,6 +526,24 @@ namespace DshComputerUse.Worker
|
|
|
438
526
|
public static extern IntPtr OpenProcess(uint access, bool inherit, uint pid);
|
|
439
527
|
[DllImport("kernel32.dll", SetLastError = true)]
|
|
440
528
|
public static extern bool CloseHandle(IntPtr h);
|
|
529
|
+
// ---- process ancestry: is a covered window the target app's OWN embedded renderer? ----
|
|
530
|
+
// A WeChat article panel (WeChatAppEx) or an Electron utility process is a legitimate
|
|
531
|
+
// descendant of the app the caller aimed at; an unrelated window on top is not.
|
|
532
|
+
public const uint TH32CS_SNAPPROCESS = 0x00000002;
|
|
533
|
+
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
|
|
534
|
+
public struct PROCESSENTRY32W
|
|
535
|
+
{
|
|
536
|
+
public uint dwSize, cntUsage, th32ProcessID;
|
|
537
|
+
public IntPtr th32DefaultHeapID;
|
|
538
|
+
public uint th32ModuleID, cntThreads, th32ParentProcessID, pcPriClassBase, dwFlags;
|
|
539
|
+
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 260)] public string szExeFile;
|
|
540
|
+
}
|
|
541
|
+
[DllImport("kernel32.dll", SetLastError = true)]
|
|
542
|
+
public static extern IntPtr CreateToolhelp32Snapshot(uint flags, uint pid);
|
|
543
|
+
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
544
|
+
public static extern bool Process32FirstW(IntPtr snapshot, ref PROCESSENTRY32W entry);
|
|
545
|
+
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
546
|
+
public static extern bool Process32NextW(IntPtr snapshot, ref PROCESSENTRY32W entry);
|
|
441
547
|
// ---- host guard: which executable is the process that owns this window? ----
|
|
442
548
|
// OpenProcess + QueryFullProcessImageName is the ONLY safe way to ask: Process.MainModule
|
|
443
549
|
// throws for any process this worker may not read (see IsHostPid).
|
|
@@ -447,6 +553,10 @@ namespace DshComputerUse.Worker
|
|
|
447
553
|
public static extern bool OpenProcessToken(IntPtr proc, uint access, out IntPtr token);
|
|
448
554
|
[DllImport("advapi32.dll", SetLastError = true)]
|
|
449
555
|
public static extern bool GetTokenInformation(IntPtr token, int cls, out uint info, uint len, out uint ret);
|
|
556
|
+
[DllImport("advapi32.dll", EntryPoint = "GetTokenInformation", SetLastError = true)]
|
|
557
|
+
public static extern bool GetTokenInformationBuffer(IntPtr token, int cls, IntPtr info, uint len, out uint ret);
|
|
558
|
+
[DllImport("advapi32.dll")] public static extern IntPtr GetSidSubAuthorityCount(IntPtr sid);
|
|
559
|
+
[DllImport("advapi32.dll")] public static extern IntPtr GetSidSubAuthority(IntPtr sid, uint index);
|
|
450
560
|
public const uint PROCESS_QUERY_LIMITED_INFORMATION = 0x1000, TOKEN_QUERY = 0x0008;
|
|
451
561
|
public const int TokenElevation = 20;
|
|
452
562
|
[DllImport("user32.dll")]
|
|
@@ -473,6 +583,8 @@ namespace DshComputerUse.Worker
|
|
|
473
583
|
public static extern IntPtr CreateDIBSection(IntPtr dc, ref BITMAPINFO bmi, uint usage, out IntPtr bits, IntPtr hSection, uint offset);
|
|
474
584
|
[DllImport("user32.dll")]
|
|
475
585
|
public static extern bool GetCursorInfo(ref CURSORINFO pci);
|
|
586
|
+
[DllImport("user32.dll", EntryPoint = "LoadCursorW")]
|
|
587
|
+
public static extern IntPtr LoadCursor(IntPtr instance, IntPtr resource);
|
|
476
588
|
[DllImport("user32.dll")]
|
|
477
589
|
public static extern bool DrawIconEx(IntPtr hdc, int xLeft, int yTop, IntPtr hIcon, int cxWidth, int cyHeight, int istepIfAniCur, IntPtr hbrFlickerFreeDraw, uint diFlags);
|
|
478
590
|
[DllImport("user32.dll")]
|
|
@@ -484,6 +596,10 @@ namespace DshComputerUse.Worker
|
|
|
484
596
|
public static extern IntPtr OpenInputDesktop(uint flags, bool inherit, uint access);
|
|
485
597
|
[DllImport("user32.dll")]
|
|
486
598
|
public static extern bool CloseDesktop(IntPtr h);
|
|
599
|
+
[DllImport("user32.dll")]
|
|
600
|
+
public static extern IntPtr GetThreadDesktop(uint threadId);
|
|
601
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
|
602
|
+
public static extern bool GetUserObjectInformation(IntPtr obj, int index, StringBuilder info, int length, out int needed);
|
|
487
603
|
|
|
488
604
|
// ---- low-level keyboard hook: the human's emergency brake ----
|
|
489
605
|
public delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
@@ -572,7 +688,6 @@ namespace DshComputerUse.Worker
|
|
|
572
688
|
[DllImport("user32.dll")] static extern bool SetWindowDisplayAffinity(IntPtr hwnd, uint affinity);
|
|
573
689
|
|
|
574
690
|
static Thread _thread;
|
|
575
|
-
static volatile bool _acting;
|
|
576
691
|
static volatile bool _preparing;
|
|
577
692
|
static volatile int _preparationStart;
|
|
578
693
|
static int _preparationSerial, _presentedPreparation, _completedPreparation;
|
|
@@ -595,9 +710,12 @@ namespace DshComputerUse.Worker
|
|
|
595
710
|
"lastSuccessAgeMs", Volatile.Read(ref _presentSuccesses) == 0 ? (object)null : unchecked((uint)(now - _lastPresentTick)),
|
|
596
711
|
"lastFailedSerial", Volatile.Read(ref _lastFailedSerial), "lastFailedWant", Volatile.Read(ref _lastFailedWant),
|
|
597
712
|
"lastWin32Error", Volatile.Read(ref _lastFailedSerial) < 0 ? (object)null : _lastFailedError,
|
|
598
|
-
"rebuiltSerial", Volatile.Read(ref _rebuiltSerial));
|
|
713
|
+
"rebuiltSerial", Volatile.Read(ref _rebuiltSerial), "pointerFault", AgentPointer.Fault);
|
|
599
714
|
}
|
|
600
|
-
|
|
715
|
+
static readonly Stopwatch PreparationClock = Stopwatch.StartNew();
|
|
716
|
+
static readonly PreparationCadence PreparationFrequency = new PreparationCadence();
|
|
717
|
+
static volatile int _preparationMs = PreparationCadence.FullMs;
|
|
718
|
+
public static int PreparationMs { get { return _preparationMs; } }
|
|
601
719
|
static volatile bool _enabled = true;
|
|
602
720
|
static volatile string _label = "AI 正在操控此电脑 · 监听中";
|
|
603
721
|
|
|
@@ -605,22 +723,36 @@ namespace DshComputerUse.Worker
|
|
|
605
723
|
public static void Touch(int holdMs)
|
|
606
724
|
{
|
|
607
725
|
if (!_enabled) return;
|
|
726
|
+
_sceneVisible = true;
|
|
608
727
|
_thinking = true;
|
|
609
728
|
EnsureStarted();
|
|
610
729
|
}
|
|
611
|
-
|
|
612
|
-
|
|
730
|
+
// Renderer visibility is deliberately separate from scene ownership.
|
|
731
|
+
// Hiding the indicator must neither release the scene nor resurrect it
|
|
732
|
+
// on the next frame solely because the authoritative hold is still true.
|
|
733
|
+
static volatile bool _sceneVisible = true;
|
|
734
|
+
public static void Off() { _sceneVisible = false; _preparing = false; _thinking = false; }
|
|
735
|
+
public static bool Acting { get { return SceneControl.Acting; } }
|
|
736
|
+
public static bool Holding { get { return SceneControl.Holding; } }
|
|
613
737
|
public static bool Preparing { get { return _preparing; } }
|
|
614
738
|
public static bool PreparationVisible { get { return _preparing && Volatile.Read(ref _presentedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
615
739
|
public static bool PreparationComplete { get { return _preparing && Volatile.Read(ref _completedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
616
|
-
public static void ActuationBegin() {
|
|
617
|
-
public static void ActuationEnd() {
|
|
740
|
+
public static void ActuationBegin() { _sceneVisible = true; EnsureStarted(); }
|
|
741
|
+
public static void ActuationEnd() { }
|
|
618
742
|
public static void Prepare(bool on)
|
|
619
743
|
{
|
|
620
744
|
if (!on) { _preparing = false; return; }
|
|
621
745
|
_preparationStart = 0;
|
|
746
|
+
_preparationMs = PreparationFrequency.Duration(PreparationClock.ElapsedMilliseconds);
|
|
622
747
|
Interlocked.Increment(ref _preparationSerial);
|
|
623
|
-
_preparing = true; _enabled = true; EnsureStarted();
|
|
748
|
+
_preparing = true; _enabled = true; _sceneVisible = true; EnsureStarted();
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
public static void PreparationPresented(int serial)
|
|
752
|
+
{
|
|
753
|
+
_preparationStart = Environment.TickCount;
|
|
754
|
+
PreparationFrequency.Presented(PreparationClock.ElapsedMilliseconds);
|
|
755
|
+
Volatile.Write(ref _presentedPreparation, serial);
|
|
624
756
|
}
|
|
625
757
|
|
|
626
758
|
// Nested injection sequences stay observable across drag/key/typing gaps.
|
|
@@ -629,7 +761,7 @@ namespace DshComputerUse.Worker
|
|
|
629
761
|
public static void InputBegin() { Interlocked.Increment(ref _injecting); }
|
|
630
762
|
public static void InputEnd() { if (Interlocked.Decrement(ref _injecting) < 0) _injecting = 0; }
|
|
631
763
|
public static bool Injecting { get { return _injecting > 0; } }
|
|
632
|
-
public static void KeepActing() { /*
|
|
764
|
+
public static void KeepActing() { /* Solid blue follows actual input; the separate scene hold survives it. */ }
|
|
633
765
|
|
|
634
766
|
// Emergency stop must LOOK like one. The normal fade is a slow ease (~1.6 s) which,
|
|
635
767
|
// during a panic, reads as "nothing happened" — so this snaps the overlay to zero on
|
|
@@ -637,8 +769,11 @@ namespace DshComputerUse.Worker
|
|
|
637
769
|
static volatile bool _killNow = false;
|
|
638
770
|
public static void Kill()
|
|
639
771
|
{
|
|
640
|
-
|
|
772
|
+
_sceneVisible = false; _preparing = false; _thinking = false; _flashStart = 0;
|
|
641
773
|
_killNow = true;
|
|
774
|
+
// The hands-off state is over (release/calm/brake): the countdown continuity window
|
|
775
|
+
// must not survive into the next sequence.
|
|
776
|
+
Program.ResetReannounceGrace();
|
|
642
777
|
}
|
|
643
778
|
|
|
644
779
|
// Cyan thinking/monitoring is a lifecycle latch, with no elapsed-time expiry. Off/Kill
|
|
@@ -653,6 +788,7 @@ namespace DshComputerUse.Worker
|
|
|
653
788
|
public static void Thinking(int holdMs)
|
|
654
789
|
{
|
|
655
790
|
if (!_enabled) return;
|
|
791
|
+
_sceneVisible = true;
|
|
656
792
|
_thinking = true;
|
|
657
793
|
EnsureStarted();
|
|
658
794
|
}
|
|
@@ -689,7 +825,8 @@ namespace DshComputerUse.Worker
|
|
|
689
825
|
public static int DeriveWant(bool flashOn, int committed)
|
|
690
826
|
{
|
|
691
827
|
if (Panic.Exited) return 0;
|
|
692
|
-
|
|
828
|
+
bool sceneVisible = _enabled && _sceneVisible;
|
|
829
|
+
return _askMode ? 5 : (_stoppedMode ? 4 : (Panic.HumanActive ? 6 : (_preparing ? 7 : (Acting && sceneVisible ? 1 : (Holding && sceneVisible ? 8 : (flashOn ? 2 : committed))))));
|
|
693
830
|
}
|
|
694
831
|
/// The title states the same fact as the light, derived the same way (an ended session must
|
|
695
832
|
/// not advertise "已暂停 · Ctrl+Alt+R 继续" in the window list).
|
|
@@ -697,7 +834,7 @@ namespace DshComputerUse.Worker
|
|
|
697
834
|
{
|
|
698
835
|
if (Panic.Exited) return "AI 已停止操控 · 未监听";
|
|
699
836
|
return _askMode ? "AI 正在提问 · 在 DSH 聊天里回答 · 无倒计时(Ctrl+Alt+R 撤销)"
|
|
700
|
-
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 ·
|
|
837
|
+
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 · " + (PreparationMs / 1000.0).ToString("0.0") + " 秒准备" : Acting ? "AI 正在操作 · 请暂勿移动键鼠" : Holding ? "AI 正在检查 · 请保持页面与鼠标位置" : ThinkingActive ? "可自由使用键鼠 · AI 未占用现场" : label);
|
|
701
838
|
}
|
|
702
839
|
static volatile string _shownTitle = null;
|
|
703
840
|
// What the indicator is ACTUALLY rendering right now, readable from outside via `probe`.
|
|
@@ -763,7 +900,7 @@ namespace DshComputerUse.Worker
|
|
|
763
900
|
|
|
764
901
|
/// Force the overlay into existence. Used when a stop is ADOPTED from another process:
|
|
765
902
|
/// the red border must be visible even though this process never actuated anything.
|
|
766
|
-
public static void Reveal() { _enabled = true; EnsureStarted(); }
|
|
903
|
+
public static void Reveal() { _enabled = true; _sceneVisible = true; EnsureStarted(); }
|
|
767
904
|
|
|
768
905
|
static void EnsureStarted()
|
|
769
906
|
{
|
|
@@ -853,6 +990,7 @@ namespace DshComputerUse.Worker
|
|
|
853
990
|
}
|
|
854
991
|
finally
|
|
855
992
|
{
|
|
993
|
+
AgentPointer.Dispose();
|
|
856
994
|
_hwnd = IntPtr.Zero; _shownTitle = null;
|
|
857
995
|
if (oldBmp != IntPtr.Zero && oldBmp != new IntPtr(-1)) SelectObject(memDc, oldBmp);
|
|
858
996
|
if (dib != IntPtr.Zero) DeleteObject(dib);
|
|
@@ -880,9 +1018,9 @@ namespace DshComputerUse.Worker
|
|
|
880
1018
|
_lastLoopTick = Environment.TickCount;
|
|
881
1019
|
_rendererStage = "rendering";
|
|
882
1020
|
Interlocked.Increment(ref _frames);
|
|
883
|
-
//
|
|
884
|
-
//
|
|
885
|
-
bool acting = _enabled && Acting;
|
|
1021
|
+
// Solid blue is actual input; breathing blue retains the scene between
|
|
1022
|
+
// steps. Cyan is reserved for an explicitly free desktop.
|
|
1023
|
+
bool acting = _enabled && _sceneVisible && Acting;
|
|
886
1024
|
bool thinking = ThinkingActive;
|
|
887
1025
|
bool flashOn = _flashStart != 0 && unchecked(Environment.TickCount - _flashStart) < _flashMs;
|
|
888
1026
|
|
|
@@ -893,6 +1031,7 @@ namespace DshComputerUse.Worker
|
|
|
893
1031
|
int preparationSerial = Volatile.Read(ref _preparationSerial);
|
|
894
1032
|
int want = DeriveWant(flashOn, committed);
|
|
895
1033
|
_want = want; // publish it: the state the light is showing, as a fact
|
|
1034
|
+
AgentPointer.Update(want == 1 || want == 8, want == 8);
|
|
896
1035
|
|
|
897
1036
|
// Keep the overlay's WINDOW TITLE in sync with the state, from the render loop.
|
|
898
1037
|
// When the title was only written by Stopped(), a process that INHERITED a stop
|
|
@@ -912,6 +1051,7 @@ namespace DshComputerUse.Worker
|
|
|
912
1051
|
if (_killNow)
|
|
913
1052
|
{
|
|
914
1053
|
_killNow = false;
|
|
1054
|
+
AgentPointer.Hide();
|
|
915
1055
|
alpha = 0;
|
|
916
1056
|
if (visible) { ShowWindow(hwnd, SW_HIDE); visible = false; }
|
|
917
1057
|
curBitmap = 0; committed = 0; paintedFlash = 0;
|
|
@@ -921,7 +1061,7 @@ namespace DshComputerUse.Worker
|
|
|
921
1061
|
}
|
|
922
1062
|
// want: 0 idle | 1 acting (deep blue) | 3 thinking (pale blue) | 2 flash (yellow)
|
|
923
1063
|
// 4 emergency-stopped (RED, steady — the session is dead until re-armed)
|
|
924
|
-
int needBitmap = want == 7 ? 6 : want == 6 ? 5 : want == 2 ? 2 : (want == 0 ? curBitmap : (want == 1 ? 3 : (want == 4 || want == 5 ? 4 : 1)));
|
|
1064
|
+
int needBitmap = want == 7 ? 6 : want == 6 ? 5 : want == 2 ? 2 : (want == 0 ? curBitmap : ((want == 1 || want == 8) ? 3 : (want == 4 || want == 5 ? 4 : 1)));
|
|
925
1065
|
// A REPAINT MUST REACH THE SCREEN EVEN WHEN ALPHA DOES NOT MOVE.
|
|
926
1066
|
//
|
|
927
1067
|
// The present below used to be gated on `diff != 0` alone, while the flash snaps
|
|
@@ -961,6 +1101,7 @@ namespace DshComputerUse.Worker
|
|
|
961
1101
|
flashFrame = true;
|
|
962
1102
|
}
|
|
963
1103
|
else if (want == 1) { target = FULL; breathFrame = true; }
|
|
1104
|
+
else if (want == 8) { target = HoldingAlpha(Environment.TickCount); breathFrame = true; }
|
|
964
1105
|
else if (want == 7) { target = 255; breathFrame = true; }
|
|
965
1106
|
else if (want == 4) target = FULL; // steady red, no breathing
|
|
966
1107
|
else if (want == 6)
|
|
@@ -1051,8 +1192,7 @@ namespace DshComputerUse.Worker
|
|
|
1051
1192
|
}
|
|
1052
1193
|
if (presented && want == 7 && !PreparationVisible)
|
|
1053
1194
|
{
|
|
1054
|
-
|
|
1055
|
-
Volatile.Write(ref _presentedPreparation, preparationSerial);
|
|
1195
|
+
PreparationPresented(preparationSerial);
|
|
1056
1196
|
}
|
|
1057
1197
|
if (presented && want == 7 && preparationElapsed >= PreparationMs)
|
|
1058
1198
|
Volatile.Write(ref _completedPreparation, preparationSerial);
|
|
@@ -1154,6 +1294,9 @@ namespace DshComputerUse.Worker
|
|
|
1154
1294
|
catch { /* display preference only; never changes control or capture behavior */ }
|
|
1155
1295
|
}
|
|
1156
1296
|
|
|
1297
|
+
public static int HoldingAlpha(int tick)
|
|
1298
|
+
{ return 120 + (int)Math.Round(42 * (1 - Math.Cos((unchecked((uint)tick) % 2500) * 2 * Math.PI / 2500.0))); }
|
|
1299
|
+
|
|
1157
1300
|
static void PaintBlue(IntPtr bits, int w, int h, bool intense)
|
|
1158
1301
|
{
|
|
1159
1302
|
WithGraphics(bits, w, h, delegate(System.Drawing.Graphics g) {
|
|
@@ -1449,6 +1592,7 @@ namespace DshComputerUse.Worker
|
|
|
1449
1592
|
CycleLit = false; // R2: the cycle is dead and must not be revived by anyone
|
|
1450
1593
|
_controlsReleased = true;
|
|
1451
1594
|
Disarm(); // ... and its monitors die WITH it, not at the next turn
|
|
1595
|
+
SceneControl.Reset();
|
|
1452
1596
|
try { Glow.Kill(); } catch { }
|
|
1453
1597
|
try { Glow.Ask(false); } catch { }
|
|
1454
1598
|
try { Glow.Stopped(false); } catch { }
|
|
@@ -1479,7 +1623,8 @@ namespace DshComputerUse.Worker
|
|
|
1479
1623
|
// is what made the first signed op hide a brake the human had already set.
|
|
1480
1624
|
if (_engaged || _stopped)
|
|
1481
1625
|
{
|
|
1482
|
-
|
|
1626
|
+
SceneControl.Reset();
|
|
1627
|
+
try { Glow.Kill(); } catch { }
|
|
1483
1628
|
try { Glow.Stopped(true); } catch { }
|
|
1484
1629
|
// START the overlay too: `Thinking()` was the only thing that ever did, so a
|
|
1485
1630
|
// cycle whose FIRST call arrives with a brake already standing painted nothing at
|
|
@@ -1902,7 +2047,8 @@ namespace DshComputerUse.Worker
|
|
|
1902
2047
|
bool showRed = !_exited && CycleLit;
|
|
1903
2048
|
if (showRed)
|
|
1904
2049
|
{
|
|
1905
|
-
|
|
2050
|
+
SceneControl.Reset();
|
|
2051
|
+
try { Glow.Kill(); } catch { }
|
|
1906
2052
|
try { Glow.Stopped(true); } catch { }
|
|
1907
2053
|
try { Glow.Reveal(); } catch { }
|
|
1908
2054
|
}
|
|
@@ -2053,6 +2199,12 @@ namespace DshComputerUse.Worker
|
|
|
2053
2199
|
if (m.dwExtraInfo != OwnMagic) Interlocked.Increment(ref _externalInput);
|
|
2054
2200
|
bool physical = PhysicalEvent(m.dwExtraInfo, m.flags, 3);
|
|
2055
2201
|
int message = wParam.ToInt32();
|
|
2202
|
+
if (!physical) {
|
|
2203
|
+
Interlocked.Exchange(ref _lastInjectedInputAt, HumanClock.ElapsedMilliseconds);
|
|
2204
|
+
int injectedKey = message == 0x201 || message == 0x202 ? 1 : message == 0x204 || message == 0x205 ? 2 :
|
|
2205
|
+
message == 0x207 || message == 0x208 ? 4 : message == 0x20B || message == 0x20C ? (((m.mouseData >> 16) & 0xFFFF) == 1 ? 5 : 6) : 0;
|
|
2206
|
+
if (injectedKey != 0) Handoff.NoteInjected(injectedKey);
|
|
2207
|
+
}
|
|
2056
2208
|
// Significant idle motion invalidates observations, but does not yield.
|
|
2057
2209
|
bool movement = message == 0x200 && PointerMotion.Record(m.pt.X, m.pt.Y, physical,
|
|
2058
2210
|
true, Handoff.Active, HumanClock.ElapsedMilliseconds);
|
|
@@ -2094,6 +2246,10 @@ namespace DshComputerUse.Worker
|
|
|
2094
2246
|
bool down = m == Native.WM_KEYDOWN_ || m == Native.WM_SYSKEYDOWN_;
|
|
2095
2247
|
bool up = m == Native.WM_KEYUP_ || m == Native.WM_SYSKEYUP_;
|
|
2096
2248
|
bool human = ClassifyAndTrackKey(k.vkCode, down, up, k.dwExtraInfo, k.flags);
|
|
2249
|
+
if (!human && (down || up)) {
|
|
2250
|
+
Interlocked.Exchange(ref _lastInjectedInputAt, HumanClock.ElapsedMilliseconds);
|
|
2251
|
+
Handoff.NoteInjected((int)k.vkCode);
|
|
2252
|
+
}
|
|
2097
2253
|
if (human && (down || up))
|
|
2098
2254
|
{
|
|
2099
2255
|
RecordHumanInput((int)k.vkCode, down, up);
|
|
@@ -2283,6 +2439,7 @@ namespace DshComputerUse.Worker
|
|
|
2283
2439
|
public static bool Armed { get { return _armed; } }
|
|
2284
2440
|
public static bool Exited { get { return _exited; } }
|
|
2285
2441
|
static readonly Stopwatch HumanClock = Stopwatch.StartNew();
|
|
2442
|
+
static long _lastInjectedInputAt;
|
|
2286
2443
|
static readonly HumanInputState Handoff = new HumanInputState();
|
|
2287
2444
|
static readonly object HandoffPublishLock = new object();
|
|
2288
2445
|
static string _publishedHandoff = "idle";
|
|
@@ -2296,6 +2453,7 @@ namespace DshComputerUse.Worker
|
|
|
2296
2453
|
{
|
|
2297
2454
|
var state = (Dictionary<string, object>)Handoff.Snapshot(HumanClock.ElapsedMilliseconds);
|
|
2298
2455
|
state["inputEpoch"] = InputEpoch; state["mouseTolerancePx"] = PointerHandoffTolerance.DistancePx;
|
|
2456
|
+
state["workerGeneration"] = Program.WorkerGeneration;
|
|
2299
2457
|
return state;
|
|
2300
2458
|
}
|
|
2301
2459
|
static void ResetMonitorRun() { /* Manual recovery must not erase physical input history. */ }
|
|
@@ -2308,6 +2466,18 @@ namespace DshComputerUse.Worker
|
|
|
2308
2466
|
if (desktop != IntPtr.Zero) { Native.CloseDesktop(desktop); return true; }
|
|
2309
2467
|
return false;
|
|
2310
2468
|
}
|
|
2469
|
+
static bool OnNormalInputDesktop()
|
|
2470
|
+
{
|
|
2471
|
+
// Async state may fail when the caller lacks desktop hook access.
|
|
2472
|
+
IntPtr desktop = Native.OpenInputDesktop(0, false, 0x0001 | 0x0008);
|
|
2473
|
+
if (desktop == IntPtr.Zero) return false;
|
|
2474
|
+
try {
|
|
2475
|
+
var current = new StringBuilder(256); var input = new StringBuilder(256); int needed;
|
|
2476
|
+
return Native.GetUserObjectInformation(Native.GetThreadDesktop(Native.GetCurrentThreadId()), 2, current, 512, out needed) &&
|
|
2477
|
+
Native.GetUserObjectInformation(desktop, 2, input, 512, out needed) &&
|
|
2478
|
+
String.Equals(current.ToString(), input.ToString(), StringComparison.OrdinalIgnoreCase);
|
|
2479
|
+
} finally { Native.CloseDesktop(desktop); }
|
|
2480
|
+
}
|
|
2311
2481
|
/// Polled from the monitor thread. Audits and notifies TRANSITIONS only.
|
|
2312
2482
|
static void PollSecureDesktop()
|
|
2313
2483
|
{
|
|
@@ -2330,19 +2500,17 @@ namespace DshComputerUse.Worker
|
|
|
2330
2500
|
throw error;
|
|
2331
2501
|
}
|
|
2332
2502
|
public static void Arm() { _armed = true; Handoff.Enable(CycleLit && !_exited, HumanClock.ElapsedMilliseconds); }
|
|
2333
|
-
public static void Disarm() { _armed = false; Handoff.Enable(false, HumanClock.ElapsedMilliseconds); PublishHandoff(); }
|
|
2503
|
+
public static void Disarm() { _armed = false; SceneControl.Reset(); Handoff.Enable(false, HumanClock.ElapsedMilliseconds); PublishHandoff(); }
|
|
2334
2504
|
public static void AgentActed() { _agentActAt = Environment.TickCount; }
|
|
2335
2505
|
public static void BeginActuation()
|
|
2336
2506
|
{
|
|
2337
2507
|
POINT point; if (Native.GetCursorPos(out point)) PointerMotion.Reset(point.X, point.Y);
|
|
2338
|
-
Glow.ActuationBegin();
|
|
2508
|
+
SceneControl.BeginActuation(); Glow.ActuationBegin();
|
|
2339
2509
|
Handoff.SetActing(true, HumanClock.ElapsedMilliseconds);
|
|
2340
2510
|
}
|
|
2341
|
-
public static void EndActuation()
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
Glow.ActuationEnd();
|
|
2345
|
-
}
|
|
2511
|
+
public static void EndActuation() { SceneControl.EndActuation(); Glow.ActuationEnd(); }
|
|
2512
|
+
public static void UpdateInputProtection()
|
|
2513
|
+
{ Handoff.SetActing(SceneControl.Acting || SceneControl.Holding, HumanClock.ElapsedMilliseconds); }
|
|
2346
2514
|
static bool IsBareModifier(uint vk)
|
|
2347
2515
|
{
|
|
2348
2516
|
return vk == Native.VK_SHIFT || vk == Native.VK_CONTROL || vk == Native.VK_MENU ||
|
|
@@ -2375,7 +2543,12 @@ namespace DshComputerUse.Worker
|
|
|
2375
2543
|
_publishedInputEpoch = epoch;
|
|
2376
2544
|
if (!ControlListenersEnabled && !CycleLit && phase == "idle") return;
|
|
2377
2545
|
if (phase != "idle") Glow.HumanYield();
|
|
2378
|
-
if (changed)
|
|
2546
|
+
if (changed) {
|
|
2547
|
+
var state = (Dictionary<string, object>)CooperationState();
|
|
2548
|
+
Program.AuditRecord("HUMAN-HANDOFF", "phase=" + phase + "; revision=" + state["revision"] +
|
|
2549
|
+
"; kind=" + state["lastKind"] + "; heldCount=" + state["heldCount"] +
|
|
2550
|
+
"; quietRemainingMs=" + state["quietRemainingMs"] + "; inputEpoch=" + state["inputEpoch"]);
|
|
2551
|
+
}
|
|
2379
2552
|
Notify("handoff", phase);
|
|
2380
2553
|
}
|
|
2381
2554
|
}
|
|
@@ -2386,14 +2559,37 @@ namespace DshComputerUse.Worker
|
|
|
2386
2559
|
PollSecureDesktop();
|
|
2387
2560
|
long now = HumanClock.ElapsedMilliseconds;
|
|
2388
2561
|
Handoff.Enable(_armed && CycleLit && !_exited, now);
|
|
2562
|
+
IntPtr sampleForeground = IntPtr.Zero;
|
|
2563
|
+
bool trusted = Handoff.HasHeld && CanReconcileHeld(_hooksReady, _thread != null && _thread.IsAlive, SecureDesktopActive,
|
|
2564
|
+
SceneControl.Acting, Glow.Injecting, now - Interlocked.Read(ref _lastInjectedInputAt), OnNormalInputDesktop()) &&
|
|
2565
|
+
Program.CanObserveAsyncKeys(out sampleForeground);
|
|
2566
|
+
bool swapped = Native.GetSystemMetrics(23) != 0; // SM_SWAPBUTTON
|
|
2567
|
+
int reconciled = Handoff.ReconcileReleased(delegate(int key) {
|
|
2568
|
+
bool down = HumanInputState.AsyncButtonMayBeDown(key, swapped,
|
|
2569
|
+
delegate(int physical) { return (Native.GetAsyncKeyState(physical) & 0x8000) != 0; });
|
|
2570
|
+
// A foreground/desktop switch during the sample makes zero unknown,
|
|
2571
|
+
// never evidence of release. The next tick checks the new token.
|
|
2572
|
+
return down || Native.GetForegroundWindow() != sampleForeground || !OnNormalInputDesktop();
|
|
2573
|
+
}, trusted, now);
|
|
2574
|
+
if (reconciled > 0) Program.AuditRecord("HUMAN-RELEASE-RECONCILED", "count=" + reconciled + "; source=three-normal-desktop-async-up-samples; quietMs=" + HumanInputState.IdleMs);
|
|
2389
2575
|
Handoff.Tick(now);
|
|
2390
2576
|
if (Handoff.Active) Program.ReleaseOwnedInputs();
|
|
2391
2577
|
PublishHandoff();
|
|
2392
2578
|
}
|
|
2579
|
+
static bool CanReconcileHeld(bool hooksReady, bool hookAlive, bool secure, bool acting, bool injecting, long sinceInjected, bool inputDesktop)
|
|
2580
|
+
{ return hooksReady && hookAlive && !secure && !acting && !injecting && sinceInjected >= 300 && inputDesktop; }
|
|
2393
2581
|
|
|
2394
2582
|
static RecoveryPermit _temporary;
|
|
2395
2583
|
static long _externalInput;
|
|
2396
2584
|
public static string TemporaryPauseId { get { lock (SyncLock) { return _temporary == null ? "" : _temporary.Id; } } }
|
|
2585
|
+
public static string EventPauseId(string kind)
|
|
2586
|
+
{
|
|
2587
|
+
// Handoff events are serialized under HandoffPublishLock. Lifecycle
|
|
2588
|
+
// transitions take SyncLock then publish, so their notification must
|
|
2589
|
+
// not take SyncLock in the reverse order. These two observation events
|
|
2590
|
+
// neither own nor consume temporary-pause credentials.
|
|
2591
|
+
return kind == "handoff" || kind == "secure" ? "" : TemporaryPauseId;
|
|
2592
|
+
}
|
|
2397
2593
|
public static void ObserveTemporary(string id)
|
|
2398
2594
|
{
|
|
2399
2595
|
lock (SyncLock) { if (_temporary != null && _temporary.Id == id) _temporary.Observed = true; }
|
|
@@ -2469,6 +2665,7 @@ namespace DshComputerUse.Worker
|
|
|
2469
2665
|
InputVersion = Interlocked.Read(ref _externalInput), InputTick = Native.LastInputTick() };
|
|
2470
2666
|
WriteStopFile(why + "\nsource=agent-temporary\npauseId=" + _temporary.Id);
|
|
2471
2667
|
}
|
|
2668
|
+
SceneControl.Reset();
|
|
2472
2669
|
try { Glow.Kill(); } catch { }
|
|
2473
2670
|
try { Glow.Stopped(true); } catch { } // red border + red badge: unmistakable
|
|
2474
2671
|
// THE BRAKE IS A STATE CHANGE, SO IT GETS A LINE (2026-09-14). This is the record the
|
|
@@ -2531,6 +2728,7 @@ namespace DshComputerUse.Worker
|
|
|
2531
2728
|
Program.AuditRecord("ENGAGE-ASK", "source=agent" +
|
|
2532
2729
|
"; askPid=" + Process.GetCurrentProcess().Id +
|
|
2533
2730
|
"; cycleLit=" + (CycleLit ? 1 : 0) + "; why=" + why);
|
|
2731
|
+
SceneControl.Reset();
|
|
2534
2732
|
try { Glow.Kill(); } catch { }
|
|
2535
2733
|
try { Glow.Ask(true); } catch { } // fast-breathing RED, not the steady stop
|
|
2536
2734
|
announce = true;
|
|
@@ -2707,7 +2905,12 @@ namespace DshComputerUse.Worker
|
|
|
2707
2905
|
// "calibrate" drives too — it walks the pointer across the screen to prove pixel==click.
|
|
2708
2906
|
// (selectRange / shiftClick / calibrate were found by build/test-op-classification.mjs on
|
|
2709
2907
|
// its first run: three real dispatcher ops the hand-written list had simply missed.)
|
|
2710
|
-
"calibrate"
|
|
2908
|
+
"calibrate",
|
|
2909
|
+
// "launch" starts a process: the machine changes, so it may not re-open a session the
|
|
2910
|
+
// human ended (Ctrl+Alt+Q) and it must not answer while a brake is up.
|
|
2911
|
+
"launch",
|
|
2912
|
+
// "focusField" clicks a field: it drives the machine exactly like a click.
|
|
2913
|
+
"focusField"
|
|
2711
2914
|
};
|
|
2712
2915
|
|
|
2713
2916
|
static readonly HashSet<string> ReadOnlyOps = new HashSet<string> {
|
|
@@ -2963,6 +3166,51 @@ namespace DshComputerUse.Worker
|
|
|
2963
3166
|
}
|
|
2964
3167
|
}
|
|
2965
3168
|
|
|
3169
|
+
// Only complete window-management chords may change their own foreground receiver.
|
|
3170
|
+
// Every down still passes the ordinary receiver guard. After the final accepted down,
|
|
3171
|
+
// only this chord's owned releases may bypass that guard; human/pause/cancel still win.
|
|
3172
|
+
public static class WindowManagementChord
|
|
3173
|
+
{
|
|
3174
|
+
public static bool IsExact(string canonical)
|
|
3175
|
+
{
|
|
3176
|
+
switch (canonical)
|
|
3177
|
+
{
|
|
3178
|
+
case "alt+tab": case "alt+shift+tab": case "alt+esc": case "alt+shift+esc":
|
|
3179
|
+
case "win+d": case "win+tab": case "win+left": case "win+right": case "win+up": case "win+down":
|
|
3180
|
+
case "rwin+d": case "rwin+tab": case "rwin+left": case "rwin+right": case "rwin+up": case "rwin+down":
|
|
3181
|
+
return true;
|
|
3182
|
+
default: return false;
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
|
|
3186
|
+
public static void Run(IList<ushort> keys, Action<ushort, bool> send,
|
|
3187
|
+
Action<ushort> release, Func<ushort, bool> isDown, Action hold,
|
|
3188
|
+
Action checkContinuation, Action invalidateTarget)
|
|
3189
|
+
{
|
|
3190
|
+
int acceptedDowns = 0;
|
|
3191
|
+
try
|
|
3192
|
+
{
|
|
3193
|
+
KeyboardChord.Run(keys, delegate(ushort key, bool up) {
|
|
3194
|
+
if (up && acceptedDowns == keys.Count)
|
|
3195
|
+
{
|
|
3196
|
+
checkContinuation();
|
|
3197
|
+
release(key); // ledger-backed: never releases someone else's key
|
|
3198
|
+
}
|
|
3199
|
+
else
|
|
3200
|
+
{
|
|
3201
|
+
send(key, up);
|
|
3202
|
+
if (!up) acceptedDowns++;
|
|
3203
|
+
}
|
|
3204
|
+
}, release, isDown, hold);
|
|
3205
|
+
}
|
|
3206
|
+
finally
|
|
3207
|
+
{
|
|
3208
|
+
// Even a partial chord can open system UI. No stale sticky/refocus continuation.
|
|
3209
|
+
if (acceptedDowns > 0) invalidateTarget();
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
3213
|
+
|
|
2966
3214
|
public sealed class ClipboardBytes
|
|
2967
3215
|
{
|
|
2968
3216
|
public uint Format;
|
|
@@ -3163,6 +3411,16 @@ namespace DshComputerUse.Worker
|
|
|
3163
3411
|
|
|
3164
3412
|
static class Program
|
|
3165
3413
|
{
|
|
3414
|
+
public const int ProtocolVersion = 2;
|
|
3415
|
+
public static readonly string WorkerGeneration = Guid.NewGuid().ToString("N");
|
|
3416
|
+
static Dictionary<string, object> RuntimeIdentity()
|
|
3417
|
+
{
|
|
3418
|
+
return Dict("packageVersion", Environment.GetEnvironmentVariable("DSH_CU_PACKAGE_VERSION") ?? "unreported",
|
|
3419
|
+
"buildId", Environment.GetEnvironmentVariable("DSH_CU_BUILD_ID") ?? "unreported",
|
|
3420
|
+
"sourceHash", Environment.GetEnvironmentVariable("DSH_CU_SOURCE_HASH") ?? "unreported",
|
|
3421
|
+
"binaryHash", Environment.GetEnvironmentVariable("DSH_CU_BINARY_HASH") ?? "unreported",
|
|
3422
|
+
"protocolVersion", ProtocolVersion, "workerGeneration", WorkerGeneration);
|
|
3423
|
+
}
|
|
3166
3424
|
static readonly JavaScriptSerializer Json = new JavaScriptSerializer();
|
|
3167
3425
|
static int _reqId = 0;
|
|
3168
3426
|
static readonly object OutLock = new object(); // responses AND unsolicited events share stdout
|
|
@@ -3218,6 +3476,23 @@ namespace DshComputerUse.Worker
|
|
|
3218
3476
|
static volatile bool _operationLive, _operationDispatched;
|
|
3219
3477
|
static bool _operationActing;
|
|
3220
3478
|
static string _lastAnnouncedSeries;
|
|
3479
|
+
// CONTINUOUS-OPERATION GRACE (human rule 2026-11-07): after one purple countdown the human
|
|
3480
|
+
// has been told to keep hands off. While operations keep arriving inside this sliding
|
|
3481
|
+
// window, the "do not intervene" state is CONTINUOUS, so no further countdown of any
|
|
3482
|
+
// duration is shown. Every dispatched mutation pushes the deadline forward, which is what
|
|
3483
|
+
// makes a real sequence exempt indefinitely; a gap longer than the window ends continuity
|
|
3484
|
+
// and the next action warns again. Wrap-safe comparison: both sides are TickCount deltas
|
|
3485
|
+
// bounded by the window itself.
|
|
3486
|
+
const int ReannounceGraceMs = 5000;
|
|
3487
|
+
static int _noReannounceUntil;
|
|
3488
|
+
static bool InReannounceGrace()
|
|
3489
|
+
{ return unchecked(_noReannounceUntil - Environment.TickCount) > 0; }
|
|
3490
|
+
static void ExtendReannounceGrace()
|
|
3491
|
+
{ _noReannounceUntil = unchecked(Environment.TickCount + ReannounceGraceMs); }
|
|
3492
|
+
/** Ends the continuity window. Called when the hands-off state itself ends (cycle release,
|
|
3493
|
+
* calm, brake), because a later action then genuinely starts a new sequence. */
|
|
3494
|
+
public static void ResetReannounceGrace()
|
|
3495
|
+
{ _noReannounceUntil = 0; }
|
|
3221
3496
|
static long _announcedHumanRevision, _announcedInputEpoch, _preparedInputEpoch;
|
|
3222
3497
|
static volatile int _activeRequestId, _lastCompletedRequestId;
|
|
3223
3498
|
// The seam is internal to the compiled program; no tool can replace it.
|
|
@@ -3225,6 +3500,68 @@ namespace DshComputerUse.Worker
|
|
|
3225
3500
|
return Native.SendInput((uint)inputs.Length, inputs, InputCbSize);
|
|
3226
3501
|
};
|
|
3227
3502
|
internal static Func<IntPtr> PreparationForeground = delegate { return Native.GetForegroundWindow(); };
|
|
3503
|
+
internal static Func<IntPtr> InputForeground = delegate { return TopLevel(Native.GetForegroundWindow()); };
|
|
3504
|
+
internal static Func<IntPtr> InputPointReceiver = delegate {
|
|
3505
|
+
POINT p; return Native.GetCursorPos(out p) ? TopLevel(Native.WindowFromPoint(p)) : IntPtr.Zero;
|
|
3506
|
+
};
|
|
3507
|
+
internal static Func<IntPtr, bool> InputWindowExists = delegate(IntPtr h) { return Native.IsWindow(h); };
|
|
3508
|
+
internal static Func<IntPtr, uint> InputWindowPid = delegate(IntPtr h) { return PidOf(h); };
|
|
3509
|
+
[ThreadStatic] static Action _receiverCheck;
|
|
3510
|
+
sealed class ReceiverScope : IDisposable
|
|
3511
|
+
{
|
|
3512
|
+
readonly Action previous;
|
|
3513
|
+
public ReceiverScope(Action check) { previous = _receiverCheck; check(); _receiverCheck = check; }
|
|
3514
|
+
public void Dispose() { _receiverCheck = previous; }
|
|
3515
|
+
}
|
|
3516
|
+
static string InputReceiverProblem(long expected, uint expectedPid, bool exists, uint livePid, long actual)
|
|
3517
|
+
{
|
|
3518
|
+
if (expected == 0 || !exists || livePid != expectedPid) return "observed target disappeared or changed identity";
|
|
3519
|
+
if (actual != expected) return "input receiver changed from hwnd=" + expected + " to hwnd=" + actual;
|
|
3520
|
+
return null;
|
|
3521
|
+
}
|
|
3522
|
+
|
|
3523
|
+
/// True when two windows belong to the SAME input family: the same window, another window
|
|
3524
|
+
/// of the same process, a descendant process's window (an embedded renderer), or a window
|
|
3525
|
+
/// owned/parented by the other. A modal dialog (Find/Replace, Save As, a browser file
|
|
3526
|
+
/// picker) is a DIFFERENT hwnd in the SAME process that takes the foreground while the
|
|
3527
|
+
/// operation it belongs to is still running — treating it as foreign made dialogs a
|
|
3528
|
+
/// keyboard black hole and blocked the screenshot that would have shown the question.
|
|
3529
|
+
public static bool SameInputFamily(IntPtr a, IntPtr b)
|
|
3530
|
+
{
|
|
3531
|
+
if (a == b) return true;
|
|
3532
|
+
if (a == IntPtr.Zero || b == IntPtr.Zero) return false;
|
|
3533
|
+
uint pa = PidOf(a), pb = PidOf(b);
|
|
3534
|
+
if (pa == 0 || pb == 0) return false;
|
|
3535
|
+
if (pa == pb) return true;
|
|
3536
|
+
if (ProcessDescendsFrom(unchecked((int)pb), unchecked((int)pa)) ||
|
|
3537
|
+
ProcessDescendsFrom(unchecked((int)pa), unchecked((int)pb))) return true;
|
|
3538
|
+
IntPtr x = b;
|
|
3539
|
+
for (int hop = 0; hop < 8 && x != IntPtr.Zero; hop++)
|
|
3540
|
+
{
|
|
3541
|
+
IntPtr owner = Native.GetWindow(x, 4); // GW_OWNER
|
|
3542
|
+
if (owner == a) return true;
|
|
3543
|
+
IntPtr parent = Native.GetParent(x);
|
|
3544
|
+
if (parent == a) return true;
|
|
3545
|
+
x = owner != IntPtr.Zero ? owner : parent;
|
|
3546
|
+
}
|
|
3547
|
+
return false;
|
|
3548
|
+
}
|
|
3549
|
+
|
|
3550
|
+
static IDisposable GuardReceiver(IntPtr expected, uint pid, bool keyboard)
|
|
3551
|
+
{
|
|
3552
|
+
return new ReceiverScope(delegate {
|
|
3553
|
+
IntPtr actual = keyboard ? InputForeground() : InputPointReceiver();
|
|
3554
|
+
string problem = InputReceiverProblem(expected.ToInt64(), pid, InputWindowExists(expected), InputWindowPid(expected), actual.ToInt64());
|
|
3555
|
+
if (problem != null)
|
|
3556
|
+
{
|
|
3557
|
+
// Same-family foreground for KEYBOARD only: the dialog is where the keystrokes
|
|
3558
|
+
// belong, so accept it and follow it. Mouse/point receivers keep the strict
|
|
3559
|
+
// check — a click must land on the pixel's real owner, never be retargeted.
|
|
3560
|
+
if (keyboard && SameInputFamily(expected, TopLevel(actual))) { SetSticky(TopLevel(actual)); return; }
|
|
3561
|
+
throw InputInterrupted("INPUT_TARGET_CHANGED", problem + ". Remaining input was refused; inspect any partial effect before retrying.");
|
|
3562
|
+
}
|
|
3563
|
+
});
|
|
3564
|
+
}
|
|
3228
3565
|
|
|
3229
3566
|
static Exception InputInterrupted(string code, string message)
|
|
3230
3567
|
{
|
|
@@ -3247,6 +3584,9 @@ namespace DshComputerUse.Worker
|
|
|
3247
3584
|
static void MarkMutation()
|
|
3248
3585
|
{
|
|
3249
3586
|
Panic.Check("actuation"); BeginActuation(); CheckHumanInput(); _operationDispatched = true;
|
|
3587
|
+
// Every dispatched mutation slides the grace window: a continuous sequence therefore
|
|
3588
|
+
// stays exempt indefinitely, while a real pause lets the window lapse.
|
|
3589
|
+
ExtendReannounceGrace();
|
|
3250
3590
|
}
|
|
3251
3591
|
static void BeginActuation()
|
|
3252
3592
|
{
|
|
@@ -3282,6 +3622,14 @@ namespace DshComputerUse.Worker
|
|
|
3282
3622
|
throw InputInterrupted("HUMAN_REOBSERVE", "The user changed the desktop between batch steps. Obtain fresh target evidence and continue the remaining task without replaying completed input.");
|
|
3283
3623
|
_preparedInputEpoch = Panic.InputEpoch; return;
|
|
3284
3624
|
}
|
|
3625
|
+
// Inside the sliding grace the sequence is still in progress: no countdown of any
|
|
3626
|
+
// duration, but a desktop change still refuses (the human may have taken over).
|
|
3627
|
+
if (InReannounceGrace())
|
|
3628
|
+
{
|
|
3629
|
+
if (_announcedInputEpoch != Panic.InputEpoch)
|
|
3630
|
+
throw InputInterrupted("HUMAN_REOBSERVE", "The user changed the desktop between actions. Obtain fresh target evidence and continue the remaining task without replaying completed input.");
|
|
3631
|
+
_preparedInputEpoch = Panic.InputEpoch; return;
|
|
3632
|
+
}
|
|
3285
3633
|
Panic.Check(op); CheckHumanInput();
|
|
3286
3634
|
long inputEpoch = Panic.InputEpoch;
|
|
3287
3635
|
IntPtr foreground = PreparationForeground();
|
|
@@ -3308,6 +3656,7 @@ namespace DshComputerUse.Worker
|
|
|
3308
3656
|
throw InputInterrupted("HUMAN_REOBSERVE", "The user changed the desktop during the preparation cue. No action was dispatched. Preserve the task, obtain fresh target evidence, and continue the authorized task immediately; this is not cancellation or abandonment.");
|
|
3309
3657
|
_lastAnnouncedSeries = series; _announcedHumanRevision = Panic.HumanRevision;
|
|
3310
3658
|
_announcedInputEpoch = _preparedInputEpoch = inputEpoch;
|
|
3659
|
+
ExtendReannounceGrace();
|
|
3311
3660
|
}
|
|
3312
3661
|
finally { Glow.Prepare(false); }
|
|
3313
3662
|
}
|
|
@@ -3321,11 +3670,31 @@ namespace DshComputerUse.Worker
|
|
|
3321
3670
|
}
|
|
3322
3671
|
public static bool CancelInput(int requestId)
|
|
3323
3672
|
{
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3673
|
+
return GetB(CancelInputStatus(requestId), "cancelled", false);
|
|
3674
|
+
}
|
|
3675
|
+
static Dictionary<string, object> CancelInputStatus(int requestId)
|
|
3676
|
+
{
|
|
3677
|
+
bool valid = requestId > 0;
|
|
3678
|
+
bool completed = valid && requestId <= _lastCompletedRequestId;
|
|
3679
|
+
if (valid && !completed) CancelledRequests[requestId] = true;
|
|
3680
|
+
bool active = valid && _activeRequestId == requestId;
|
|
3681
|
+
if (active) Interlocked.Increment(ref _cancelGeneration);
|
|
3682
|
+
bool cleanup = false; int owned = -1; string state = valid ? "input-lock-busy" : "invalid-request";
|
|
3683
|
+
// The reader must remain responsive even if SendInput/provider work
|
|
3684
|
+
// owns InputLock. Failure to obtain it is uncertainty, not clean-up.
|
|
3685
|
+
if (valid && Monitor.TryEnter(InputLock, 200)) {
|
|
3686
|
+
try {
|
|
3687
|
+
// Never release a newer request's accepted input on behalf
|
|
3688
|
+
// of an older cancellation.
|
|
3689
|
+
if (_activeRequestId == requestId) ReleaseOwnedInputs();
|
|
3690
|
+
owned = OwnedUps.Count;
|
|
3691
|
+
cleanup = owned == 0;
|
|
3692
|
+
state = cleanup ? "no-owned-input" : "owned-input-remains";
|
|
3693
|
+
} finally { Monitor.Exit(InputLock); }
|
|
3694
|
+
}
|
|
3695
|
+
return Dict("requestId", requestId, "cancelled", valid && !completed,
|
|
3696
|
+
"cleanupConfirmed", cleanup, "ownedInputCount", owned, "cleanupState", state,
|
|
3697
|
+
"workerGeneration", WorkerGeneration);
|
|
3329
3698
|
}
|
|
3330
3699
|
static string InputKey(INPUT input, out bool up, out INPUT release, out int physical)
|
|
3331
3700
|
{
|
|
@@ -3366,6 +3735,8 @@ namespace DshComputerUse.Worker
|
|
|
3366
3735
|
string key = InputKey(inputs[0], out isUp, out release, out physical);
|
|
3367
3736
|
if (isUp && key != null && !OwnedUps.ContainsKey(key)) return 1;
|
|
3368
3737
|
}
|
|
3738
|
+
if (_receiverCheck != null) _receiverCheck();
|
|
3739
|
+
CheckHumanInput();
|
|
3369
3740
|
uint sent = InputSender(inputs);
|
|
3370
3741
|
if (sent > 0) _operationDispatched = true;
|
|
3371
3742
|
for (int i = 0; i < Math.Min(sent, inputs.Length); i++)
|
|
@@ -3509,6 +3880,16 @@ namespace DshComputerUse.Worker
|
|
|
3509
3880
|
[STAThread]
|
|
3510
3881
|
static int Main(string[] args)
|
|
3511
3882
|
{
|
|
3883
|
+
// The cursor guardian owns only the temporary visual-hide lease. It
|
|
3884
|
+
// must never install input hooks, open a control cycle or emit worker
|
|
3885
|
+
// protocol messages into the parent's inherited output stream.
|
|
3886
|
+
if (args.Length > 0 && args[0] == "--pointer-guardian")
|
|
3887
|
+
{
|
|
3888
|
+
int ownerPid; long ownerStartTicks;
|
|
3889
|
+
if (args.Length != 4 || !Int32.TryParse(args[2], out ownerPid) ||
|
|
3890
|
+
!Int64.TryParse(args[3], out ownerStartTicks)) return 2;
|
|
3891
|
+
return PointerVisibilityLease.GuardianMain(args[1], ownerPid, ownerStartTicks);
|
|
3892
|
+
}
|
|
3512
3893
|
// DPI awareness first: it decides the coordinate space of everything below
|
|
3513
3894
|
// (GetSystemMetrics, GetCursorPos, capture pixels, UIA rects AND the
|
|
3514
3895
|
// interpretation of SendInput's absolute coordinates). Record which call won.
|
|
@@ -3537,8 +3918,9 @@ namespace DshComputerUse.Worker
|
|
|
3537
3918
|
// lives; the consumer can read the fields and draw no conclusion the worker did
|
|
3538
3919
|
// not record itself.
|
|
3539
3920
|
string hint;
|
|
3540
|
-
|
|
3541
|
-
|
|
3921
|
+
string pauseId = Panic.EventPauseId(kind);
|
|
3922
|
+
if (kind == "panic" && !string.IsNullOrEmpty(pauseId))
|
|
3923
|
+
hint = "Agent temporary diagnostic pause. Inspect the current target, then use checked recover only if the blocker is resolved. Credential: " + pauseId;
|
|
3542
3924
|
else if (kind == "panic")
|
|
3543
3925
|
hint = "STOPPED (brake engaged). The machine is stopped: nothing that drives it runs " +
|
|
3544
3926
|
"until a human resumes with Ctrl+Alt+R. WRAP UP NOW: one short summary of what " +
|
|
@@ -3563,7 +3945,7 @@ namespace DshComputerUse.Worker
|
|
|
3563
3945
|
WriteLine(Json.Serialize(Dict(
|
|
3564
3946
|
"event", kind,
|
|
3565
3947
|
"why", why,
|
|
3566
|
-
"pauseId",
|
|
3948
|
+
"pauseId", pauseId,
|
|
3567
3949
|
"cooperation", Panic.CooperationState(),
|
|
3568
3950
|
"ts", NowMs(),
|
|
3569
3951
|
"hint", hint)));
|
|
@@ -3600,6 +3982,8 @@ namespace DshComputerUse.Worker
|
|
|
3600
3982
|
Dictionary<string, object> hello = Dict(
|
|
3601
3983
|
"event", "ready",
|
|
3602
3984
|
"pid", Process.GetCurrentProcess().Id,
|
|
3985
|
+
"protocolVersion", ProtocolVersion, "workerGeneration", WorkerGeneration,
|
|
3986
|
+
"runtime", RuntimeIdentity(),
|
|
3603
3987
|
// COLD-START STATE, REPORTED AS A FACT (cold-start bug, 2026-09-13): this process has
|
|
3604
3988
|
// already adopted any STOP/EXITED record that was on disk before it existed, so the
|
|
3605
3989
|
// core can start from the machine's real state instead of "not stopped" — the two
|
|
@@ -3624,9 +4008,9 @@ namespace DshComputerUse.Worker
|
|
|
3624
4008
|
{
|
|
3625
4009
|
object argsValue;
|
|
3626
4010
|
var cancelArgs = request.TryGetValue("args", out argsValue) ? argsValue as Dictionary<string, object> : null;
|
|
3627
|
-
|
|
4011
|
+
var cancellation = CancelInputStatus(cancelArgs == null ? 0 : GetI(cancelArgs, "requestId", 0));
|
|
3628
4012
|
WriteLine(parser.Serialize(Dict("id", GetI(request, "id", 0), "ok", true,
|
|
3629
|
-
"data",
|
|
4013
|
+
"data", cancellation)));
|
|
3630
4014
|
continue;
|
|
3631
4015
|
}
|
|
3632
4016
|
} catch { /* Main loop reports malformed requests in the normal format. */ }
|
|
@@ -3856,6 +4240,8 @@ namespace DshComputerUse.Worker
|
|
|
3856
4240
|
case "type": FailsafeCheck(a); return Ok(Type(a));
|
|
3857
4241
|
case "clipRead": return Ok(ClipRead(a));
|
|
3858
4242
|
case "clipWrite": FailsafeCheck(a); return Ok(ClipWrite(a));
|
|
4243
|
+
case "launch": FailsafeCheck(a); return Ok(Launch(a));
|
|
4244
|
+
case "focusField": FailsafeCheck(a); return Ok(FocusField(a));
|
|
3859
4245
|
case "windows": return Ok(Windows(a));
|
|
3860
4246
|
case "activate": FailsafeCheck(a); return Ok(Activate(a));
|
|
3861
4247
|
case "windowOp": FailsafeCheck(a); return Ok(WindowOp(a));
|
|
@@ -3893,7 +4279,7 @@ namespace DshComputerUse.Worker
|
|
|
3893
4279
|
// Is a real injection sequence in flight RIGHT NOW? The ACTING light is driven
|
|
3894
4280
|
// by the actual action scope, so both are reported as observable facts.
|
|
3895
4281
|
"injecting", Glow.Injecting,
|
|
3896
|
-
"acting", Glow.Acting, "preparing", Glow.Preparing, "preparationMs", Glow.PreparationMs,
|
|
4282
|
+
"acting", Glow.Acting, "holding", Glow.Holding, "preparing", Glow.Preparing, "preparationMs", Glow.PreparationMs,
|
|
3897
4283
|
"title", Glow.ShownTitle == null ? "" : Glow.ShownTitle);
|
|
3898
4284
|
// EVENTS THE OS ACTUALLY INSERTED (not the plan): 0 after a plain pause with
|
|
3899
4285
|
// nothing held is the observable form of "no ghost right-click / Alt menu".
|
|
@@ -4090,6 +4476,7 @@ namespace DshComputerUse.Worker
|
|
|
4090
4476
|
IntPtr old = Native.SelectObject(mem, dib);
|
|
4091
4477
|
// SRCCOPY | CAPTUREBLT so layered windows are included
|
|
4092
4478
|
Native.BitBlt(mem, 0, 0, w, h, screen, x, y, 0x00CC0020 | 0x40000000);
|
|
4479
|
+
long capturedAt = NowMs();
|
|
4093
4480
|
bool cursorDrawn = false;
|
|
4094
4481
|
int cursorFlags = 0, cursorAtX = 0, cursorAtY = 0;
|
|
4095
4482
|
if (withCursor)
|
|
@@ -4152,13 +4539,16 @@ namespace DshComputerUse.Worker
|
|
|
4152
4539
|
ep.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, q);
|
|
4153
4540
|
outBmp.Save(ms, codec, ep);
|
|
4154
4541
|
}
|
|
4542
|
+
int imageWidth = outBmp.Width, imageHeight = outBmp.Height;
|
|
4155
4543
|
outBmp.Dispose();
|
|
4156
4544
|
POINT cp2; Native.GetCursorPos(out cp2);
|
|
4157
4545
|
Dictionary<string, object> res = Dict(
|
|
4158
4546
|
"image", Convert.ToBase64String(ms.ToArray()),
|
|
4159
4547
|
"mime", mime,
|
|
4160
4548
|
"region", Dict("x", x, "y", y, "width", w, "height", h),
|
|
4161
|
-
"scaledTo",
|
|
4549
|
+
"scaledTo", imageWidth,
|
|
4550
|
+
"imageSize", new int[] { imageWidth, imageHeight },
|
|
4551
|
+
"capturedAt", capturedAt,
|
|
4162
4552
|
"cursor", PtDict(cp2.X, cp2.Y),
|
|
4163
4553
|
"ts", NowMs());
|
|
4164
4554
|
if (withCursor)
|
|
@@ -4170,7 +4560,8 @@ namespace DshComputerUse.Worker
|
|
|
4170
4560
|
res["cursorDrawn"] = cursorDrawn;
|
|
4171
4561
|
res["cursorFlags"] = cursorFlags;
|
|
4172
4562
|
res["cursorScreenPos"] = PtDict(cursorAtX, cursorAtY);
|
|
4173
|
-
res["cursorInImage"] =
|
|
4563
|
+
res["cursorInImage"] = Dict("x", (cursorAtX - x) * (double)imageWidth / w,
|
|
4564
|
+
"y", (cursorAtY - y) * (double)imageHeight / h);
|
|
4174
4565
|
res["cursorNote"] = cursorDrawn
|
|
4175
4566
|
? "pointer burned into the image at cursorInImage"
|
|
4176
4567
|
: ((cursorFlags & 1) == 0
|
|
@@ -4316,6 +4707,46 @@ namespace DshComputerUse.Worker
|
|
|
4316
4707
|
SendInputChecked(new INPUT[] { inp });
|
|
4317
4708
|
}
|
|
4318
4709
|
|
|
4710
|
+
internal static string StandardCursorShape(IntPtr cursor, Func<int, IntPtr> load)
|
|
4711
|
+
{
|
|
4712
|
+
if (cursor == IntPtr.Zero) return "unknown";
|
|
4713
|
+
int[] ids = { 32644, 32645, 32643, 32642, 32646, 32512, 32513, 32649, 32514, 32515, 32648, 32650, 32651 };
|
|
4714
|
+
string[] names = { "resize-ew", "resize-ns", "resize-nesw", "resize-nwse", "move", "arrow", "ibeam", "hand", "wait", "cross", "not-allowed", "app-starting", "help" };
|
|
4715
|
+
for (int i = 0; i < ids.Length; i++) if (load(ids[i]) == cursor) return names[i];
|
|
4716
|
+
return "unknown"; // Custom/browser-drawn cursors do not reveal a CSS value.
|
|
4717
|
+
}
|
|
4718
|
+
|
|
4719
|
+
static bool CursorMatches(string expected, string actual, bool visible)
|
|
4720
|
+
{
|
|
4721
|
+
if (String.IsNullOrEmpty(expected)) return true;
|
|
4722
|
+
if (!visible || actual == "unknown") return false;
|
|
4723
|
+
return expected == "resize" ? actual.StartsWith("resize-", StringComparison.Ordinal) : expected == actual;
|
|
4724
|
+
}
|
|
4725
|
+
|
|
4726
|
+
static Dictionary<string, object> CursorEvidence(bool available, bool visible, IntPtr cursor, Func<int, IntPtr> load)
|
|
4727
|
+
{
|
|
4728
|
+
visible = available && visible;
|
|
4729
|
+
string shape = visible ? StandardCursorShape(cursor, load) : "unknown";
|
|
4730
|
+
string classification = !available ? "unavailable" : !visible ? "not-visible" : shape == "unknown" ? "unrecognized" : "standard";
|
|
4731
|
+
var result = Dict("available", available, "visible", visible, "shape", shape,
|
|
4732
|
+
"classification", classification, "source", "win32-standard-cursor");
|
|
4733
|
+
if (classification == "unrecognized")
|
|
4734
|
+
result["note"] = "Visible cursor handle does not match a supported Windows standard cursor. Browser col-resize/row-resize and custom cursors may be unrecognized; this does not prove a hit or a miss on a draggable handle.";
|
|
4735
|
+
return result;
|
|
4736
|
+
}
|
|
4737
|
+
|
|
4738
|
+
static Dictionary<string, object> CursorSnapshot()
|
|
4739
|
+
{
|
|
4740
|
+
CURSORINFO ci = new CURSORINFO();
|
|
4741
|
+
ci.cbSize = (uint)Marshal.SizeOf(typeof(CURSORINFO));
|
|
4742
|
+
bool available = Native.GetCursorInfo(ref ci);
|
|
4743
|
+
bool visible = available && (ci.flags & Native.CURSOR_SHOWING) != 0;
|
|
4744
|
+
var result = CursorEvidence(available, visible, ci.hCursor,
|
|
4745
|
+
delegate(int id) { return Native.LoadCursor(IntPtr.Zero, new IntPtr(id)); });
|
|
4746
|
+
result["screenPos"] = available ? (object)PtDict(ci.ptScreenPos.X, ci.ptScreenPos.Y) : null;
|
|
4747
|
+
return result;
|
|
4748
|
+
}
|
|
4749
|
+
|
|
4319
4750
|
static Dictionary<string, object> Move(Dictionary<string, object> a)
|
|
4320
4751
|
{
|
|
4321
4752
|
int x = GetI(a, "x", int.MinValue), y = GetI(a, "y", int.MinValue);
|
|
@@ -4337,11 +4768,75 @@ namespace DshComputerUse.Worker
|
|
|
4337
4768
|
Native.GetCursorPos(out p);
|
|
4338
4769
|
}
|
|
4339
4770
|
CheckHumanInput();
|
|
4340
|
-
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections);
|
|
4771
|
+
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections, "cursor", CursorSnapshot());
|
|
4772
|
+
}
|
|
4773
|
+
|
|
4774
|
+
static Rectangle ClickSampleRegion(int x, int y, Rectangle desktop)
|
|
4775
|
+
{
|
|
4776
|
+
int width = Math.Min(340, desktop.Width), height = Math.Min(340, desktop.Height);
|
|
4777
|
+
if (width <= 0 || height <= 0) return Rectangle.Empty;
|
|
4778
|
+
int left = (int)Math.Min(Math.Max((long)x - width / 2, desktop.Left), (long)desktop.Right - width);
|
|
4779
|
+
int top = (int)Math.Min(Math.Max((long)y - height / 2, desktop.Top), (long)desktop.Bottom - height);
|
|
4780
|
+
return new Rectangle(left, top, width, height);
|
|
4781
|
+
}
|
|
4782
|
+
|
|
4783
|
+
static int VerificationNumber(Dictionary<string, object> rectangle, string key)
|
|
4784
|
+
{
|
|
4785
|
+
object raw;
|
|
4786
|
+
if (!rectangle.TryGetValue(key, out raw) || !(raw is int || raw is long || raw is double || raw is float || raw is decimal))
|
|
4787
|
+
throw new ArgumentException("missing integer " + key);
|
|
4788
|
+
double n = Convert.ToDouble(raw);
|
|
4789
|
+
if (Double.IsNaN(n) || Double.IsInfinity(n) || n != Math.Truncate(n) || n <= Int32.MinValue || n > Int32.MaxValue)
|
|
4790
|
+
throw new ArgumentException("invalid integer " + key);
|
|
4791
|
+
return (int)n;
|
|
4792
|
+
}
|
|
4793
|
+
static Rectangle VerificationSampleRegion(Dictionary<string, object> a, int x, int y, Rectangle desktop)
|
|
4794
|
+
{
|
|
4795
|
+
if (!a.ContainsKey("verificationRegion")) return ClickSampleRegion(x, y, desktop);
|
|
4796
|
+
try {
|
|
4797
|
+
var box = a["verificationRegion"] as Dictionary<string, object>;
|
|
4798
|
+
if (box == null) throw new ArgumentException("expected rectangle object");
|
|
4799
|
+
int rx = VerificationNumber(box, "x"), ry = VerificationNumber(box, "y");
|
|
4800
|
+
int rw = VerificationNumber(box, "width"), rh = VerificationNumber(box, "height");
|
|
4801
|
+
if (rw <= 0 || rh <= 0 || (long)rw * rh > 16777216 ||
|
|
4802
|
+
(long)rx + rw > Int32.MaxValue || (long)ry + rh > Int32.MaxValue)
|
|
4803
|
+
throw new ArgumentException("positive size, 16M pixel maximum, non-overflowing edges required");
|
|
4804
|
+
Rectangle requested = new Rectangle(rx, ry, rw, rh);
|
|
4805
|
+
Rectangle visible = Rectangle.Intersect(requested, desktop);
|
|
4806
|
+
if (visible.Width <= 0 || visible.Height <= 0) throw new ArgumentException("region is outside the virtual desktop");
|
|
4807
|
+
return visible;
|
|
4808
|
+
} catch (Exception error) {
|
|
4809
|
+
var refusal = new ArgumentException("INVALID_VERIFICATION_REGION: " + error.Message + "; no input was dispatched.");
|
|
4810
|
+
refusal.Data["code"] = "INVALID_VERIFICATION_REGION"; refusal.Data["outcome"] = "not-dispatched";
|
|
4811
|
+
throw refusal;
|
|
4812
|
+
}
|
|
4813
|
+
}
|
|
4814
|
+
static void AddPixelVerification(Dictionary<string, object> result, Dictionary<string, object> args,
|
|
4815
|
+
Rectangle region, byte[] before, byte[] after)
|
|
4816
|
+
{
|
|
4817
|
+
result["effectVerified"] = false;
|
|
4818
|
+
result["verificationAvailable"] = before != null && after != null && before.Length > 0 && before.Length == after.Length;
|
|
4819
|
+
result["verificationSource"] = args.ContainsKey("verificationRegion") ? "requested-region" : "click-neighborhood";
|
|
4820
|
+
result["verificationRegion"] = Dict("x", region.X, "y", region.Y, "width", region.Width, "height", region.Height);
|
|
4821
|
+
if (args.ContainsKey("verificationRegion")) result["requestedVerificationRegion"] = args["verificationRegion"];
|
|
4822
|
+
if (!(bool)result["verificationAvailable"]) {
|
|
4823
|
+
result["verificationNote"] = "Pixel samples unavailable; input completion does not establish the intended effect.";
|
|
4824
|
+
return;
|
|
4825
|
+
}
|
|
4826
|
+
double changed = Math.Round(SigDiffPct(before, after, 8), 2);
|
|
4827
|
+
result["changedPct"] = changed;
|
|
4828
|
+
result["verificationNote"] = "Sampled pixel difference only, not semantic verification. Inspect the expected result before replaying input.";
|
|
4829
|
+
if (changed < 0.15 && !args.ContainsKey("verify"))
|
|
4830
|
+
result["warning"] = "No sampled change in the " + (args.ContainsKey("verificationRegion") ? "requested witness region" : "click neighborhood") +
|
|
4831
|
+
". This is expected for an already-focused or already-satisfied control; an effect may also be delayed or elsewhere. " +
|
|
4832
|
+
"Check the intended state and focus/receiver before retrying.";
|
|
4341
4833
|
}
|
|
4342
4834
|
|
|
4343
4835
|
static Dictionary<string, object> Click(Dictionary<string, object> a)
|
|
4344
4836
|
{
|
|
4837
|
+
Rectangle sampleRegion = VerificationSampleRegion(a, GetI(a, "x", 0), GetI(a, "y", 0), new Rectangle(
|
|
4838
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
4839
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
4345
4840
|
string btn = GetS(a, "button", "left");
|
|
4346
4841
|
int clicks = Math.Max(1, Math.Min(3, GetI(a, "clicks", 1)));
|
|
4347
4842
|
int gap = GetI(a, "intervalMs", 60);
|
|
@@ -4451,7 +4946,20 @@ namespace DshComputerUse.Worker
|
|
|
4451
4946
|
return dry;
|
|
4452
4947
|
}
|
|
4453
4948
|
if (hostVictim) throw new Exception(HostGuardMsg("click", unchecked((int)PidOf(under))));
|
|
4454
|
-
|
|
4949
|
+
bool allowCovered = GetB(a, "allowCovered", false);
|
|
4950
|
+
// NARROWED (2026-11-07): allowCovered is an escape for the target app's OWN embedded
|
|
4951
|
+
// renderer only. It used to authorise the click whatever owned the pixel, so an
|
|
4952
|
+
// unrelated front window could receive input meant for the target app — measured
|
|
4953
|
+
// live when a covered click landed on the agent's own GUI instead of WeChat. An
|
|
4954
|
+
// ancestor test cannot be spoofed by z-order: the receiver must be the target's
|
|
4955
|
+
// descendant process (WeChatAppEx under Weixin.exe, an Electron utility process).
|
|
4956
|
+
if (allowCovered && blocked == "COVERED" &&
|
|
4957
|
+
!ProcessDescendsFrom(unchecked((int)PidOf(under)), unchecked((int)PidOf(want))))
|
|
4958
|
+
throw new Exception("precondition failed (COVERED-UNRELATED): " + detail +
|
|
4959
|
+
". allowCovered only authorises the target application's own embedded renderer " +
|
|
4960
|
+
"(a descendant of the target process); an unrelated window on top is never clicked. " +
|
|
4961
|
+
"NOTHING was clicked. Bring the target into view, or aim at a pixel inside it.");
|
|
4962
|
+
if (blocked != null && !allowCovered)
|
|
4455
4963
|
throw new Exception("precondition failed (" + blocked + "): " + detail + ". NOTHING was clicked. " +
|
|
4456
4964
|
"Bring the target into view first (activate / move / restore), or pass allowCovered:true to click anyway. " +
|
|
4457
4965
|
"Use computer_uia at:\"" + tx + "," + ty + "\" for the full ground truth.");
|
|
@@ -4471,41 +4979,33 @@ namespace DshComputerUse.Worker
|
|
|
4471
4979
|
// The window under the click point becomes the sticky target for later typing.
|
|
4472
4980
|
try { SetSticky(under); } catch { }
|
|
4473
4981
|
}
|
|
4474
|
-
//
|
|
4475
|
-
//
|
|
4476
|
-
// missed pixel, wrong element) — and "no change" is exactly the silent failure that
|
|
4477
|
-
// used to be indistinguishable from success. Region sampling is ~1700 pixels, cheaper
|
|
4478
|
-
// than a screenshot by three orders of magnitude.
|
|
4479
|
-
int vsW = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
4480
|
-
int vsH = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
4982
|
+
// A cheap pixel witness, not a business-success condition. The caller may
|
|
4983
|
+
// identify an effect elsewhere (e.g. a player bar), independently of the click.
|
|
4481
4984
|
bool verify = (tx != int.MinValue) && GetB(a, "verify", true);
|
|
4482
|
-
int
|
|
4483
|
-
int bx = Math.Min(Math.Max(tx - half, margin), Math.Max(margin, vsW - margin - 2 * half));
|
|
4484
|
-
int by = Math.Min(Math.Max(ty - half, margin), Math.Max(margin, vsH - margin - 2 * half));
|
|
4485
|
-
int bw = Math.Min(2 * half, Math.Max(16, vsW - 2 * margin));
|
|
4486
|
-
int bh = Math.Min(2 * half, Math.Max(16, vsH - 2 * margin));
|
|
4985
|
+
int bx = sampleRegion.X, by = sampleRegion.Y, bw = sampleRegion.Width, bh = sampleRegion.Height;
|
|
4487
4986
|
byte[] before = verify ? FrameSignatureRect(bx, by, bw, bh, 8) : null;
|
|
4488
4987
|
|
|
4489
4988
|
// Re-ask at the moment of injection: the pointer has travelled since the precondition
|
|
4490
4989
|
// above, and the only promise that matters is about the pixel the button is really
|
|
4491
4990
|
// pressed on. Cheap (one WindowFromPoint) and it closes the window between check and use.
|
|
4492
4991
|
if (tx != int.MinValue) HostGuardPoint(tx, ty, "click");
|
|
4992
|
+
if (under == IntPtr.Zero) under = InputPointReceiver();
|
|
4493
4993
|
|
|
4494
4994
|
uint down = Native.MOUSEEVENTF_LEFTDOWN, up = Native.MOUSEEVENTF_LEFTUP;
|
|
4495
4995
|
if (btn == "right") { down = Native.MOUSEEVENTF_RIGHTDOWN; up = Native.MOUSEEVENTF_RIGHTUP; }
|
|
4496
4996
|
else if (btn == "middle") { down = Native.MOUSEEVENTF_MIDDLEDOWN; up = Native.MOUSEEVENTF_MIDDLEUP; }
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4997
|
+
using (GuardReceiver(under, InputWindowPid(under), false)) {
|
|
4998
|
+
for (int i = 0; i < clicks; i++) {
|
|
4999
|
+
SendMouse(down, 0, 0, 0); Thread.Sleep(30); SendMouse(up, 0, 0, 0);
|
|
5000
|
+
if (i < clicks - 1) Thread.Sleep(gap);
|
|
5001
|
+
}
|
|
4501
5002
|
}
|
|
4502
5003
|
EndActuationForReadback();
|
|
4503
|
-
|
|
5004
|
+
byte[] after = null;
|
|
4504
5005
|
if (verify && before != null)
|
|
4505
5006
|
{
|
|
4506
5007
|
Thread.Sleep(GetI(a, "verifyDelayMs", 240));
|
|
4507
|
-
|
|
4508
|
-
if (after != null) changedPct = Math.Round(SigDiffPct(before, after, 8), 2);
|
|
5008
|
+
after = FrameSignatureRect(bx, by, bw, bh, 8);
|
|
4509
5009
|
}
|
|
4510
5010
|
POINT fp; Native.GetCursorPos(out fp);
|
|
4511
5011
|
Dictionary<string, object> clickFocus = FocusReport(a, false);
|
|
@@ -4515,18 +5015,58 @@ namespace DshComputerUse.Worker
|
|
|
4515
5015
|
if (coveredBy != null) clickOut["coveredBy"] = coveredBy;
|
|
4516
5016
|
if (under != IntPtr.Zero)
|
|
4517
5017
|
clickOut["receiver"] = Short(WinTitleOf(under), 40) + " (pid " + PidOf(under) + ")";
|
|
4518
|
-
if (
|
|
4519
|
-
{
|
|
4520
|
-
clickOut["changedPct"] = changedPct;
|
|
4521
|
-
if (changedPct < 0.15 && !a.ContainsKey("verify"))
|
|
4522
|
-
clickOut["warning"] = "nothing changed near the click point within ~240ms. This is EXPECTED when the " +
|
|
4523
|
-
"click lands on an already-focused editor or a control already in the desired state (a caret blink " +
|
|
4524
|
-
"alone sits below the threshold — test report 2026-11-04); it can also mean a dead/disabled control " +
|
|
4525
|
-
"or a wrong pixel. Read focus/receiver above and only retry if they disagree with the intent.";
|
|
4526
|
-
}
|
|
5018
|
+
if (verify) AddPixelVerification(clickOut, a, sampleRegion, before, after);
|
|
4527
5019
|
return clickOut;
|
|
4528
5020
|
}
|
|
4529
5021
|
|
|
5022
|
+
static Exception DragPrecondition(string code, string detail)
|
|
5023
|
+
{
|
|
5024
|
+
var error = InputInterrupted(code, detail + " No drag button was pressed; earlier focus or pointer movement may have occurred.");
|
|
5025
|
+
error.Data["dragStarted"] = false;
|
|
5026
|
+
return error;
|
|
5027
|
+
}
|
|
5028
|
+
|
|
5029
|
+
static Dictionary<string, object> CheckDragCursor(string expected, Dictionary<string, object> actual)
|
|
5030
|
+
{
|
|
5031
|
+
expected = (expected ?? "").Trim().ToLowerInvariant();
|
|
5032
|
+
string shape = (string)actual["shape"];
|
|
5033
|
+
string classification = (string)actual["classification"];
|
|
5034
|
+
string status = expected.Length == 0 ? "not-requested" : CursorMatches(expected, shape, (bool)actual["visible"])
|
|
5035
|
+
? "matched" : classification == "standard" ? "mismatched" : "unverifiable";
|
|
5036
|
+
var check = Dict("status", status, "expected", expected.Length == 0 ? null : (object)expected, "observed", shape);
|
|
5037
|
+
if (status == "mismatched" || status == "unverifiable") {
|
|
5038
|
+
string detail = status == "mismatched"
|
|
5039
|
+
? "Expected " + expected + ", observed standard cursor " + shape + ". Re-observe the intended handle before retrying."
|
|
5040
|
+
: "Cannot verify expected cursor " + expected + ": classification=" + classification + ". An unknown/unavailable cursor is not proof of an incorrect target. Browser col-resize/row-resize may use nonstandard cursors. If fresh independent geometry and hit-testing confirm the handle, the optional expectCursor may be omitted; verify the application result afterward.";
|
|
5041
|
+
// Keep the existing error code for callers; cursorCheck distinguishes a known
|
|
5042
|
+
// shape disagreement from missing evidence. Neither is converted to a match.
|
|
5043
|
+
var error = DragPrecondition("CURSOR_MISMATCH", detail);
|
|
5044
|
+
error.Data["expectedCursor"] = expected;
|
|
5045
|
+
error.Data["actualCursor"] = actual;
|
|
5046
|
+
error.Data["cursorCheck"] = check;
|
|
5047
|
+
throw error;
|
|
5048
|
+
}
|
|
5049
|
+
return check;
|
|
5050
|
+
}
|
|
5051
|
+
|
|
5052
|
+
static IntPtr CheckDragStart(Dictionary<string, object> a, int x, int y)
|
|
5053
|
+
{
|
|
5054
|
+
HostGuardPoint(x, y, "drag");
|
|
5055
|
+
POINT point; point.X = x; point.Y = y;
|
|
5056
|
+
IntPtr under = TopLevel(Native.WindowFromPoint(point));
|
|
5057
|
+
IntPtr want = a.ContainsKey("expectHwnd") ? new IntPtr(GetI(a, "expectHwnd", 0)) : _stickyHwnd;
|
|
5058
|
+
if (under == IntPtr.Zero) throw DragPrecondition("DRAG_START_UNREACHABLE", "No window owns the drag start pixel.");
|
|
5059
|
+
if (want != IntPtr.Zero) {
|
|
5060
|
+
if (!Native.IsWindow(want) || Native.IsIconic(want) || IsOffScreen(want))
|
|
5061
|
+
throw DragPrecondition("DRAG_TARGET_UNAVAILABLE", "The expected drag window is gone, minimized or off-screen.");
|
|
5062
|
+
if (under != want && PidOf(under) != PidOf(want))
|
|
5063
|
+
throw DragPrecondition("DRAG_START_COVERED", "The start pixel belongs to hwnd=" + under.ToInt64() + ", not expected hwnd=" + want.ToInt64() + ". Re-observe the target.");
|
|
5064
|
+
}
|
|
5065
|
+
if (ProcessElevated(PidOf(under)) && !SelfElevated())
|
|
5066
|
+
throw DragPrecondition("DRAG_START_ELEVATED", "The drag receiver is elevated and this worker cannot inject into it.");
|
|
5067
|
+
return under;
|
|
5068
|
+
}
|
|
5069
|
+
|
|
4530
5070
|
static Dictionary<string, object> Drag(Dictionary<string, object> a)
|
|
4531
5071
|
{
|
|
4532
5072
|
int fx = GetI(a, "fromX", int.MinValue), fy = GetI(a, "fromY", int.MinValue);
|
|
@@ -4541,9 +5081,22 @@ namespace DshComputerUse.Worker
|
|
|
4541
5081
|
int durMs = Math.Max(50, GetI(a, "durationMs", 350));
|
|
4542
5082
|
AgentActing(a, "drag");
|
|
4543
5083
|
CheckHumanInput();
|
|
5084
|
+
if (a.ContainsKey("expectHwnd")) {
|
|
5085
|
+
HostGuardNamed(a, "drag");
|
|
5086
|
+
FocusReport(a, true);
|
|
5087
|
+
}
|
|
5088
|
+
CheckDragStart(a, fx, fy);
|
|
4544
5089
|
AnimatedMove(fx, fy, GetI(a, "moveDurationMs", 250));
|
|
4545
5090
|
Thread.Sleep(80);
|
|
4546
|
-
|
|
5091
|
+
CheckHumanInput();
|
|
5092
|
+
POINT startPoint;
|
|
5093
|
+
if (!Native.GetCursorPos(out startPoint) || Dist(startPoint, fx, fy) > 2)
|
|
5094
|
+
throw DragPrecondition("DRAG_START_MISSED", "The pointer did not reach the measured drag start within two screen pixels.");
|
|
5095
|
+
IntPtr startWindow = CheckDragStart(a, startPoint.X, startPoint.Y);
|
|
5096
|
+
Dictionary<string, object> startCursor = CursorSnapshot();
|
|
5097
|
+
Dictionary<string, object> cursorCheck = CheckDragCursor(GetS(a, "expectCursor", ""), startCursor);
|
|
5098
|
+
using (GuardReceiver(startWindow, InputWindowPid(startWindow), false))
|
|
5099
|
+
SendMouse(Native.MOUSEEVENTF_LEFTDOWN, 0, 0, 0);
|
|
4547
5100
|
Thread.Sleep(80);
|
|
4548
5101
|
POINT probe0; Native.GetCursorPos(out probe0);
|
|
4549
5102
|
if (Dist(probe0, fx, fy) > 24)
|
|
@@ -4572,7 +5125,9 @@ namespace DshComputerUse.Worker
|
|
|
4572
5125
|
SendMouse(Native.MOUSEEVENTF_LEFTUP, 0, 0, 0);
|
|
4573
5126
|
POINT p; Native.GetCursorPos(out p);
|
|
4574
5127
|
CheckHumanInput();
|
|
4575
|
-
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections
|
|
5128
|
+
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections,
|
|
5129
|
+
"dragStarted", true, "startCursor", startCursor, "cursorCheck", cursorCheck, "startHwnd", startWindow.ToInt64(),
|
|
5130
|
+
"effectVerified", false, "note", "Input sequence completed. Cursor/window checks do not prove that a handle moved; verify the intended UI change.");
|
|
4576
5131
|
}
|
|
4577
5132
|
|
|
4578
5133
|
static Dictionary<string, object> Scroll(Dictionary<string, object> a)
|
|
@@ -4587,10 +5142,14 @@ namespace DshComputerUse.Worker
|
|
|
4587
5142
|
else HostGuardCursor("scroll");
|
|
4588
5143
|
AgentActing(a, "scroll");
|
|
4589
5144
|
if (a.ContainsKey("x") && a.ContainsKey("y")) Move(a);
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
5145
|
+
IntPtr scrollTarget = InputPointReceiver();
|
|
5146
|
+
if (_stickyHwnd != IntPtr.Zero && scrollTarget != _stickyHwnd && !SameInputFamily(_stickyHwnd, scrollTarget))
|
|
5147
|
+
throw InputInterrupted("INPUT_TARGET_CHANGED", "Scroll receiver differs from the observed target. No wheel input was sent.");
|
|
5148
|
+
using (GuardReceiver(scrollTarget, InputWindowPid(scrollTarget), false)) {
|
|
5149
|
+
for (int i = 0; i < lines; i++) {
|
|
5150
|
+
SendMouse(flag, 0, 0, unchecked((uint)(sign * (int)Native.WHEEL_DELTA)));
|
|
5151
|
+
Thread.Sleep(25);
|
|
5152
|
+
}
|
|
4594
5153
|
}
|
|
4595
5154
|
POINT p; Native.GetCursorPos(out p);
|
|
4596
5155
|
return PtDict(p.X, p.Y);
|
|
@@ -4634,6 +5193,14 @@ namespace DshComputerUse.Worker
|
|
|
4634
5193
|
|
|
4635
5194
|
static Dictionary<string, object> Indicator(Dictionary<string, object> a)
|
|
4636
5195
|
{
|
|
5196
|
+
if (a.ContainsKey("hold"))
|
|
5197
|
+
{
|
|
5198
|
+
SceneControl.SetHolding(GetB(a, "hold", false));
|
|
5199
|
+
if (SceneControl.Holding) Glow.Reveal();
|
|
5200
|
+
if (Panic.CycleLit && !Panic.Exited) Glow.Thinking(0);
|
|
5201
|
+
return Dict("holding", Glow.Holding, "cycleActive", Panic.CycleLit && !Panic.Exited,
|
|
5202
|
+
"indicator", Glow.Holding ? "holding" : "free");
|
|
5203
|
+
}
|
|
4637
5204
|
bool on = GetB(a, "on", true);
|
|
4638
5205
|
bool soft = GetB(a, "soft", false);
|
|
4639
5206
|
if (on) { Glow.Enabled = true; Glow.Touch(GetI(a, "ms", 30000)); }
|
|
@@ -4717,20 +5284,38 @@ namespace DshComputerUse.Worker
|
|
|
4717
5284
|
{
|
|
4718
5285
|
int x = GetI(a, "x", int.MinValue), y = GetI(a, "y", int.MinValue);
|
|
4719
5286
|
if (x == int.MinValue || y == int.MinValue) throw new Exception("x,y required");
|
|
5287
|
+
Rectangle witness = VerificationSampleRegion(a, x, y, new Rectangle(
|
|
5288
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
5289
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
4720
5290
|
HostGuardPoint(x, y, "shiftClick");
|
|
5291
|
+
POINT targetPoint; targetPoint.X = x; targetPoint.Y = y;
|
|
5292
|
+
IntPtr targetWindow = TopLevel(Native.WindowFromPoint(targetPoint));
|
|
5293
|
+
if (targetWindow == IntPtr.Zero) throw InputInterrupted("INPUT_TARGET_CHANGED", "Shift-click target is unavailable; no input sent.");
|
|
5294
|
+
uint targetPid = InputWindowPid(targetWindow);
|
|
4721
5295
|
int clicks = Math.Max(1, Math.Min(3, GetI(a, "clicks", 1)));
|
|
4722
5296
|
AgentActing(a, "shiftClick");
|
|
4723
5297
|
CheckHumanInput();
|
|
4724
5298
|
AnimatedMove(x, y, Math.Max(60, GetI(a, "moveDurationMs", 200)));
|
|
4725
5299
|
Thread.Sleep(GetI(a, "settleMs", 90));
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
5300
|
+
byte[] before = a.ContainsKey("verificationRegion") ? FrameSignatureRect(witness.X, witness.Y, witness.Width, witness.Height, 8) : null;
|
|
5301
|
+
using (GuardReceiver(targetWindow, targetPid, false)) {
|
|
5302
|
+
SendKey(0x10, false); // VK_SHIFT down
|
|
5303
|
+
Thread.Sleep(40);
|
|
5304
|
+
ClickTimes(clicks);
|
|
5305
|
+
Thread.Sleep(40);
|
|
5306
|
+
SendKey(0x10, true); // VK_SHIFT up
|
|
5307
|
+
}
|
|
4731
5308
|
Thread.Sleep(GetI(a, "settleMs", 90));
|
|
4732
5309
|
CheckHumanInput();
|
|
4733
|
-
|
|
5310
|
+
var result = Dict("x", x, "y", y, "clicks", clicks);
|
|
5311
|
+
if (a.ContainsKey("verificationRegion")) {
|
|
5312
|
+
EndActuationForReadback();
|
|
5313
|
+
Thread.Sleep(240);
|
|
5314
|
+
byte[] after = FrameSignatureRect(witness.X, witness.Y, witness.Width, witness.Height, 8);
|
|
5315
|
+
CheckHumanInput();
|
|
5316
|
+
AddPixelVerification(result, a, witness, before, after);
|
|
5317
|
+
}
|
|
5318
|
+
return result;
|
|
4734
5319
|
}
|
|
4735
5320
|
|
|
4736
5321
|
static void ClickTimes(int n)
|
|
@@ -4819,7 +5404,11 @@ namespace DshComputerUse.Worker
|
|
|
4819
5404
|
IntPtr dib = Native.CreateDIBSection(screen, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
4820
5405
|
if (dib == IntPtr.Zero) { Native.DeleteDC(mem); Native.ReleaseDC(IntPtr.Zero, screen); return null; }
|
|
4821
5406
|
IntPtr old = Native.SelectObject(mem, dib);
|
|
4822
|
-
Native.BitBlt(mem, 0, 0, rw, rh, screen, rx, ry, 0x00CC0020 | 0x40000000)
|
|
5407
|
+
if (!Native.BitBlt(mem, 0, 0, rw, rh, screen, rx, ry, 0x00CC0020 | 0x40000000)) {
|
|
5408
|
+
Native.SelectObject(mem, old);
|
|
5409
|
+
Native.DeleteObject(dib); Native.DeleteDC(mem); Native.ReleaseDC(IntPtr.Zero, screen);
|
|
5410
|
+
return null;
|
|
5411
|
+
}
|
|
4823
5412
|
int n = ((rw - 1) / step + 1) * ((rh - 1) / step + 1);
|
|
4824
5413
|
byte[] sig = new byte[n];
|
|
4825
5414
|
int k = 0;
|
|
@@ -5242,6 +5831,35 @@ namespace DshComputerUse.Worker
|
|
|
5242
5831
|
}
|
|
5243
5832
|
throw new Exception("unknown key: " + k);
|
|
5244
5833
|
}
|
|
5834
|
+
static string NormalizeKeyCombo(string combo)
|
|
5835
|
+
{
|
|
5836
|
+
if (String.IsNullOrWhiteSpace(combo)) throw new Exception("key combo required");
|
|
5837
|
+
var ordered = new List<string>(); var seen = new HashSet<string>(StringComparer.Ordinal);
|
|
5838
|
+
foreach (string raw in combo.Split('+')) {
|
|
5839
|
+
string key = raw.Trim().ToLowerInvariant();
|
|
5840
|
+
if (key.Length == 0) throw new Exception("key combo contains an empty token; use plus for the + key");
|
|
5841
|
+
switch (key) {
|
|
5842
|
+
case "control": key = "ctrl"; break;
|
|
5843
|
+
case "lwin": case "windows": case "meta": key = "win"; break;
|
|
5844
|
+
case "return": key = "enter"; break;
|
|
5845
|
+
case "escape": key = "esc"; break;
|
|
5846
|
+
case "bksp": key = "backspace"; break;
|
|
5847
|
+
case "del": key = "delete"; break;
|
|
5848
|
+
case "ins": key = "insert"; break;
|
|
5849
|
+
case "pageup": key = "pgup"; break;
|
|
5850
|
+
case "pagedown": key = "pgdn"; break;
|
|
5851
|
+
case "capslock": key = "caps"; break;
|
|
5852
|
+
case "printscreen": key = "prtsc"; break;
|
|
5853
|
+
}
|
|
5854
|
+
VkOf(key); // parser and native dispatch accept the same key set
|
|
5855
|
+
if (seen.Add(key)) ordered.Add(key);
|
|
5856
|
+
}
|
|
5857
|
+
var result = new List<string>();
|
|
5858
|
+
foreach (string modifier in new string[] { "ctrl", "alt", "shift", "win", "rwin" })
|
|
5859
|
+
if (seen.Contains(modifier)) { result.Add(modifier); ordered.Remove(modifier); }
|
|
5860
|
+
result.AddRange(ordered);
|
|
5861
|
+
return String.Join("+", result);
|
|
5862
|
+
}
|
|
5245
5863
|
|
|
5246
5864
|
static void SendKey(ushort vk, bool up)
|
|
5247
5865
|
{
|
|
@@ -5281,11 +5899,11 @@ namespace DshComputerUse.Worker
|
|
|
5281
5899
|
|
|
5282
5900
|
static Dictionary<string, object> Key(Dictionary<string, object> a)
|
|
5283
5901
|
{
|
|
5902
|
+
if (_receiverCheck != null) _receiverCheck(); // nested paste may not silently repair a stolen receiver
|
|
5284
5903
|
string combo = GetS(a, "combo", null);
|
|
5285
5904
|
if (combo == null) throw new Exception("combo required (e.g. 'ctrl+shift+t', 'enter')");
|
|
5286
|
-
|
|
5287
|
-
|
|
5288
|
-
if (keys.Count == 0) throw new Exception("empty combo");
|
|
5905
|
+
combo = NormalizeKeyCombo(combo);
|
|
5906
|
+
List<string> keys = new List<string>(combo.Split('+'));
|
|
5289
5907
|
AgentActing(a, "key");
|
|
5290
5908
|
// Keyboard input goes to the FOREGROUND window — re-assert the window the agent is
|
|
5291
5909
|
// working in, and report which window actually received the keys.
|
|
@@ -5300,18 +5918,38 @@ namespace DshComputerUse.Worker
|
|
|
5300
5918
|
HostGuardForeground("key");
|
|
5301
5919
|
List<ushort> vks = new List<ushort>();
|
|
5302
5920
|
foreach (string k in keys) { ushort vk = VkOf(k); if (!vks.Contains(vk)) vks.Add(vk); }
|
|
5303
|
-
|
|
5304
|
-
|
|
5305
|
-
|
|
5921
|
+
bool windowManagement = WindowManagementChord.IsExact(combo);
|
|
5922
|
+
using (GuardReceiver(new IntPtr(Convert.ToInt64(focus["hwnd"])), Convert.ToUInt32(focus["pid"]), true))
|
|
5923
|
+
{
|
|
5924
|
+
if (windowManagement)
|
|
5925
|
+
WindowManagementChord.Run(vks, SendKey, ReleaseKeyForCleanup,
|
|
5926
|
+
delegate(ushort vk) { return (Native.GetAsyncKeyState(vk) & 0x8000) != 0; },
|
|
5927
|
+
delegate { WaitForKeyHold(Math.Max(20, GetI(a, "holdMs", 60))); },
|
|
5928
|
+
delegate { WaitForKeyHold(0); }, InvalidateKeyboardTarget);
|
|
5929
|
+
else
|
|
5930
|
+
KeyboardChord.Run(vks, SendKey, ReleaseKeyForCleanup,
|
|
5931
|
+
delegate(ushort vk) { return (Native.GetAsyncKeyState(vk) & 0x8000) != 0; },
|
|
5932
|
+
delegate { WaitForKeyHold(Math.Max(20, GetI(a, "holdMs", 60))); });
|
|
5933
|
+
}
|
|
5306
5934
|
EndActuationForReadback();
|
|
5307
5935
|
Thread.Sleep(30);
|
|
5308
|
-
Dictionary<string, object> keyOut = Dict("combo", combo,
|
|
5936
|
+
Dictionary<string, object> keyOut = AddFocusReadback(Dict("combo", combo), a, focus);
|
|
5937
|
+
if (windowManagement)
|
|
5938
|
+
{
|
|
5939
|
+
var after = (Dictionary<string, object>)keyOut["focusAfter"];
|
|
5940
|
+
keyOut["receiverChanged"] = Convert.ToInt64(focus["hwnd"]) != Convert.ToInt64(after["hwnd"]) ||
|
|
5941
|
+
Convert.ToUInt32(focus["pid"]) != Convert.ToUInt32(after["pid"]);
|
|
5942
|
+
keyOut["reobserveRequired"] = true;
|
|
5943
|
+
keyOut["effectVerified"] = false;
|
|
5944
|
+
keyOut["hint"] = KeyboardReobserveHint;
|
|
5945
|
+
}
|
|
5309
5946
|
if (_lastRefocus != null) keyOut["refocused"] = _lastRefocus;
|
|
5310
5947
|
return keyOut;
|
|
5311
5948
|
}
|
|
5312
5949
|
|
|
5313
5950
|
static Dictionary<string, object> Type(Dictionary<string, object> a)
|
|
5314
5951
|
{
|
|
5952
|
+
if (_receiverCheck != null) _receiverCheck();
|
|
5315
5953
|
string text = GetS(a, "text", null);
|
|
5316
5954
|
if (text == null) throw new Exception("text required");
|
|
5317
5955
|
AgentActing(a, "type");
|
|
@@ -5320,6 +5958,7 @@ namespace DshComputerUse.Worker
|
|
|
5320
5958
|
HostGuardNamed(a, "type");
|
|
5321
5959
|
Dictionary<string, object> focus = FocusReport(a, true);
|
|
5322
5960
|
HostGuardForeground("type"); // same rule as `key`: the foreground window is the victim
|
|
5961
|
+
using (GuardReceiver(new IntPtr(Convert.ToInt64(focus["hwnd"])), Convert.ToUInt32(focus["pid"]), true)) {
|
|
5323
5962
|
// ...and whether whatever owns the focus can hold text at all. Typing into a control
|
|
5324
5963
|
// with no editable pattern is silently discarded, or interpreted as shortcuts.
|
|
5325
5964
|
string typeWarn = null;
|
|
@@ -5372,7 +6011,7 @@ namespace DshComputerUse.Worker
|
|
|
5372
6011
|
downgraded["clipboardWarning"] = "The previous clipboard could not be preserved (private or oversized formats) and was overwritten by this paste. Restore it manually if it mattered.";
|
|
5373
6012
|
if (typeWarn != null) downgraded["warning"] = typeWarn;
|
|
5374
6013
|
if (_lastRefocus != null) downgraded["refocused"] = _lastRefocus;
|
|
5375
|
-
return downgraded;
|
|
6014
|
+
return AddFocusReadback(downgraded, a, focus);
|
|
5376
6015
|
}
|
|
5377
6016
|
Exception inputError = null;
|
|
5378
6017
|
try
|
|
@@ -5406,7 +6045,7 @@ namespace DshComputerUse.Worker
|
|
|
5406
6045
|
pr["clipboardWarning"] = "Clipboard changed or could not be restored. Verify the target value before repeating input.";
|
|
5407
6046
|
if (typeWarn != null) pr["warning"] = typeWarn;
|
|
5408
6047
|
if (_lastRefocus != null) pr["refocused"] = _lastRefocus;
|
|
5409
|
-
return pr;
|
|
6048
|
+
return AddFocusReadback(pr, a, focus);
|
|
5410
6049
|
}
|
|
5411
6050
|
int charDelay = GetI(a, "charDelayMs", 8);
|
|
5412
6051
|
foreach (char ch in text)
|
|
@@ -5430,10 +6069,17 @@ namespace DshComputerUse.Worker
|
|
|
5430
6069
|
SendInputChecked(new INPUT[] { down, up });
|
|
5431
6070
|
Thread.Sleep(charDelay);
|
|
5432
6071
|
}
|
|
5433
|
-
Dictionary<string, object> tr = Dict("mode", "unicode", "chars", text.Length
|
|
6072
|
+
Dictionary<string, object> tr = Dict("mode", "unicode", "chars", text.Length);
|
|
5434
6073
|
if (typeWarn != null) tr["warning"] = typeWarn;
|
|
5435
6074
|
if (_lastRefocus != null) tr["refocused"] = _lastRefocus;
|
|
5436
|
-
return tr;
|
|
6075
|
+
return AddFocusReadback(tr, a, focus);
|
|
6076
|
+
}
|
|
6077
|
+
}
|
|
6078
|
+
static Dictionary<string, object> AddFocusReadback(Dictionary<string, object> result, Dictionary<string, object> args, Dictionary<string, object> before)
|
|
6079
|
+
{
|
|
6080
|
+
var after = FocusReport(args, false);
|
|
6081
|
+
result["focusBefore"] = before; result["focusAfter"] = after; result["focus"] = after;
|
|
6082
|
+
return result;
|
|
5437
6083
|
}
|
|
5438
6084
|
|
|
5439
6085
|
// ---------- clipboard ----------
|
|
@@ -5454,6 +6100,103 @@ namespace DshComputerUse.Worker
|
|
|
5454
6100
|
return Dict("written", true, "chars", text.Length);
|
|
5455
6101
|
}
|
|
5456
6102
|
|
|
6103
|
+
// ---------- launch: open an app / URL / folder / document in ONE op ----------
|
|
6104
|
+
// WHY THIS IS AN OP AND NOT A SIDE-EFFECT IN THE PLUGIN (2026-11-07, measured): the
|
|
6105
|
+
// plugin's first version spawned explorer from Node. It worked, but it never touched the
|
|
6106
|
+
// worker — so the status halo never lit, the human got no signal that the machine had just
|
|
6107
|
+
// been acted on, and audit/rate/cooperation had to be re-implemented by hand. This op puts
|
|
6108
|
+
// launch on the SAME funnel as every other actuation.
|
|
6109
|
+
//
|
|
6110
|
+
// RESOLUTION IS ShellExecute's, not ours: ProcessStartInfo.UseShellExecute resolves App
|
|
6111
|
+
// Paths aliases ("calc"), shell: verbs, URLs, folders and document associations exactly
|
|
6112
|
+
// like the Run dialog. Measured on this machine: `explorer.exe calc` opens a FILE EXPLORER
|
|
6113
|
+
// window (a bare word is a location to explorer), while ShellExecute("calc") starts the
|
|
6114
|
+
// Calculator — so the launcher, not a shell wrapper, is the correct resolver.
|
|
6115
|
+
//
|
|
6116
|
+
// INJECTION: the target crosses as ONE Executable/FileName value. No command line is
|
|
6117
|
+
// composed, no shell interpreter is involved, and nothing is appended after the target —
|
|
6118
|
+
// the same argv-boundary argument the plugin's validate() documents.
|
|
6119
|
+
static Dictionary<string, object> Launch(Dictionary<string, object> a)
|
|
6120
|
+
{
|
|
6121
|
+
string target = GetS(a, "target", "");
|
|
6122
|
+
if (target.Length == 0) throw new Exception("LAUNCH_BAD_TARGET: target is empty");
|
|
6123
|
+
// Optional arguments (2026-11-07). A dedicated browser profile is a LAUNCHER argument,
|
|
6124
|
+
// not part of the target: without this the only route to `msedge.exe --user-data-dir=…`
|
|
6125
|
+
// was Win+R, which the input guard reports as INPUT_TARGET_CHANGED after the chord has
|
|
6126
|
+
// already landed and then refuses screenshots of the dialog it just opened.
|
|
6127
|
+
string arguments = LaunchArguments(a);
|
|
6128
|
+
// Same gate as every other mutation: brake/pause/human-input checks run BEFORE the
|
|
6129
|
+
// process is started, and the deep steady blue says "the machine is being acted on".
|
|
6130
|
+
AgentActing(a, "launch");
|
|
6131
|
+
MarkMutation();
|
|
6132
|
+
System.Diagnostics.Process started = null;
|
|
6133
|
+
try
|
|
6134
|
+
{
|
|
6135
|
+
var info = new System.Diagnostics.ProcessStartInfo(target) { UseShellExecute = true };
|
|
6136
|
+
if (arguments.Length != 0) info.Arguments = arguments;
|
|
6137
|
+
started = System.Diagnostics.Process.Start(info);
|
|
6138
|
+
}
|
|
6139
|
+
catch (Exception e)
|
|
6140
|
+
{
|
|
6141
|
+
throw new Exception("LAUNCH_FAILED: " + e.Message);
|
|
6142
|
+
}
|
|
6143
|
+
// A launch returns in milliseconds; a 5 ms flash is invisible to a human. Hold the lit
|
|
6144
|
+
// border for a short, SELF-EXPIRING tail (Glow.Touch) instead of a latch, so this can
|
|
6145
|
+
// never claim the machine is still being driven after the action has finished.
|
|
6146
|
+
int tailMs = GetI(a, "tailMs", 2500);
|
|
6147
|
+
if (tailMs > 0)
|
|
6148
|
+
{
|
|
6149
|
+
tailMs = Math.Min(10000, tailMs);
|
|
6150
|
+
Glow.Enabled = true;
|
|
6151
|
+
Glow.Touch(tailMs);
|
|
6152
|
+
}
|
|
6153
|
+
int pid = 0;
|
|
6154
|
+
try { if (started != null) pid = started.Id; } catch { pid = 0; }
|
|
6155
|
+
return Dict("launched", true, "target", target, "pid", pid, "glowMs", Math.Max(0, tailMs));
|
|
6156
|
+
}
|
|
6157
|
+
|
|
6158
|
+
/// Quote ONE argument the way CommandLineToArgvW parses it back. Quoting happens here,
|
|
6159
|
+
/// exactly once: the caller passes an array of values, never a pre-built command line.
|
|
6160
|
+
static string QuoteArgument(string s)
|
|
6161
|
+
{
|
|
6162
|
+
if (s.Length > 0 && s.IndexOfAny(new[] { ' ', '\t', '"' }) < 0) return s;
|
|
6163
|
+
var sb = new StringBuilder();
|
|
6164
|
+
sb.Append('"');
|
|
6165
|
+
int backslashes = 0;
|
|
6166
|
+
foreach (char c in s)
|
|
6167
|
+
{
|
|
6168
|
+
if (c == '\\') { backslashes++; continue; }
|
|
6169
|
+
if (c == '"') { sb.Append('\\', backslashes * 2 + 1); sb.Append('"'); backslashes = 0; continue; }
|
|
6170
|
+
if (backslashes > 0) { sb.Append('\\', backslashes); backslashes = 0; }
|
|
6171
|
+
sb.Append(c);
|
|
6172
|
+
}
|
|
6173
|
+
if (backslashes > 0) sb.Append('\\', backslashes * 2);
|
|
6174
|
+
sb.Append('"');
|
|
6175
|
+
return sb.ToString();
|
|
6176
|
+
}
|
|
6177
|
+
|
|
6178
|
+
/// Validate the optional `args` array and turn it into one Windows command line.
|
|
6179
|
+
static string LaunchArguments(Dictionary<string, object> a)
|
|
6180
|
+
{
|
|
6181
|
+
object raw;
|
|
6182
|
+
if (!a.TryGetValue("args", out raw) || raw == null) return "";
|
|
6183
|
+
if (raw is string) throw new Exception("LAUNCH_BAD_ARGS: args must be an array of strings, not a string");
|
|
6184
|
+
var list = raw as System.Collections.IEnumerable;
|
|
6185
|
+
if (list == null) throw new Exception("LAUNCH_BAD_ARGS: args must be an array of strings");
|
|
6186
|
+
var parts = new List<string>();
|
|
6187
|
+
foreach (object item in list)
|
|
6188
|
+
{
|
|
6189
|
+
if (item == null) throw new Exception("LAUNCH_BAD_ARGS: null argument");
|
|
6190
|
+
string s = Convert.ToString(item);
|
|
6191
|
+
if (s.Length == 0) throw new Exception("LAUNCH_BAD_ARGS: empty argument");
|
|
6192
|
+
if (s.Length > 2000) throw new Exception("LAUNCH_BAD_ARGS: argument longer than 2000 characters");
|
|
6193
|
+
if (s.IndexOf('\0') >= 0) throw new Exception("LAUNCH_BAD_ARGS: NUL in argument");
|
|
6194
|
+
parts.Add(QuoteArgument(s));
|
|
6195
|
+
}
|
|
6196
|
+
if (parts.Count > 64) throw new Exception("LAUNCH_BAD_ARGS: more than 64 arguments");
|
|
6197
|
+
return string.Join(" ", parts);
|
|
6198
|
+
}
|
|
6199
|
+
|
|
5457
6200
|
// ---------- windows ----------
|
|
5458
6201
|
static Dictionary<string, object> RectDict(RECT r)
|
|
5459
6202
|
{
|
|
@@ -5492,7 +6235,34 @@ namespace DshComputerUse.Worker
|
|
|
5492
6235
|
list.Add(WinDict(h));
|
|
5493
6236
|
return true;
|
|
5494
6237
|
}, IntPtr.Zero);
|
|
5495
|
-
return Dict("windows", list)
|
|
6238
|
+
return Dict("windows", list, "virtualScreen", VirtualScreenDict(),
|
|
6239
|
+
"cooperation", Panic.CooperationState(), "indicator", IndicatorSnapshot());
|
|
6240
|
+
}
|
|
6241
|
+
|
|
6242
|
+
static string IndicatorStateName(int want)
|
|
6243
|
+
{
|
|
6244
|
+
switch (want) {
|
|
6245
|
+
case 0: return "idle";
|
|
6246
|
+
case 1: return "acting";
|
|
6247
|
+
case 2: return "capture";
|
|
6248
|
+
case 3: return "thinking";
|
|
6249
|
+
case 4: return "paused";
|
|
6250
|
+
case 5: return "asking";
|
|
6251
|
+
case 6: return "human-handoff";
|
|
6252
|
+
case 7: return "preparing";
|
|
6253
|
+
case 8: return "reserved";
|
|
6254
|
+
default: return "unknown";
|
|
6255
|
+
}
|
|
6256
|
+
}
|
|
6257
|
+
|
|
6258
|
+
static Dictionary<string, object> IndicatorSnapshot()
|
|
6259
|
+
{
|
|
6260
|
+
// Want/title describe the last rendered frame, not a prediction that
|
|
6261
|
+
// an input will run. The cooperation record supplies the handoff cause.
|
|
6262
|
+
int want = Glow.Want;
|
|
6263
|
+
return Dict("state", IndicatorStateName(want), "want", want,
|
|
6264
|
+
"title", Glow.ShownTitle ?? "", "injecting", Glow.Injecting,
|
|
6265
|
+
"acting", Glow.Acting, "preparing", Glow.Preparing, "holding", Glow.Holding);
|
|
5496
6266
|
}
|
|
5497
6267
|
|
|
5498
6268
|
static IntPtr FindHwnd(Dictionary<string, object> a)
|
|
@@ -5550,15 +6320,25 @@ namespace DshComputerUse.Worker
|
|
|
5550
6320
|
// guard, which must re-assert the same window before typing).
|
|
5551
6321
|
string how = RefocusWindow(h, GetB(a, "clickFallback", true));
|
|
5552
6322
|
uint confirmedPid = 0;
|
|
5553
|
-
|
|
6323
|
+
long modalOf = 0;
|
|
6324
|
+
IntPtr foreground = Native.GetForegroundWindow();
|
|
6325
|
+
// A MODAL DIALOG of the same app becomes the real input receiver (Find/Replace, Save As,
|
|
6326
|
+
// a print dialog). Confirming the parent instead left every later key and screenshot
|
|
6327
|
+
// refused while the app was literally waiting for an answer.
|
|
6328
|
+
IntPtr dialog = (foreground != IntPtr.Zero && foreground != h && SameInputFamily(h, foreground)) ? foreground : IntPtr.Zero;
|
|
6329
|
+
IntPtr confirmed = dialog != IntPtr.Zero ? dialog : h;
|
|
6330
|
+
if (how == null || !Native.IsWindow(h) || (confirmedPid = PidOf(confirmed)) == 0 || !IsForeground(confirmed)) how = "failed";
|
|
5554
6331
|
else
|
|
5555
6332
|
{
|
|
5556
|
-
|
|
5557
|
-
|
|
6333
|
+
if (dialog != IntPtr.Zero) modalOf = h.ToInt64();
|
|
6334
|
+
SetSticky(confirmed);
|
|
6335
|
+
_kickoffFocus.Confirm(confirmed.ToInt64(), confirmedPid);
|
|
5558
6336
|
}
|
|
5559
6337
|
CheckHumanInput();
|
|
6338
|
+
if (how != "failed") _keyboardReobserveRequired = false;
|
|
5560
6339
|
return Dict("activated", how != "failed", "activatedVia", how,
|
|
5561
|
-
"hwnd",
|
|
6340
|
+
"hwnd", confirmed.ToInt64(), "pid", confirmedPid == 0 ? null : (object)confirmedPid,
|
|
6341
|
+
"modalOf", modalOf == 0 ? null : (object)modalOf,
|
|
5562
6342
|
"hint", how == "failed"
|
|
5563
6343
|
? "foreground lock refused every method; click the window (or its taskbar button) and retry"
|
|
5564
6344
|
: null);
|
|
@@ -5578,6 +6358,22 @@ namespace DshComputerUse.Worker
|
|
|
5578
6358
|
// 2. if that window belongs to an ELEVATED process while we are not, UIPI silently
|
|
5579
6359
|
// drops every event — say so loudly instead of reporting a successful no-op.
|
|
5580
6360
|
static IntPtr _stickyHwnd = IntPtr.Zero;
|
|
6361
|
+
static volatile bool _keyboardReobserveRequired;
|
|
6362
|
+
const string KeyboardReobserveHint = "A window-management chord was dispatched; this is not proof that a switch succeeded. " +
|
|
6363
|
+
"Inspect computer_state {windows:true}, then use computer_window {hwnd:...,op:'activate'} and require activated:true before further keyboard input.";
|
|
6364
|
+
static void InvalidateKeyboardTarget()
|
|
6365
|
+
{
|
|
6366
|
+
_stickyHwnd = IntPtr.Zero;
|
|
6367
|
+
ResetKickoffFocus();
|
|
6368
|
+
_keyboardReobserveRequired = true;
|
|
6369
|
+
}
|
|
6370
|
+
static void CheckKeyboardReobserve()
|
|
6371
|
+
{
|
|
6372
|
+
if (!_keyboardReobserveRequired) return;
|
|
6373
|
+
var error = NotDispatched("INPUT_REOBSERVE_REQUIRED", "INPUT_REOBSERVE_REQUIRED: " + KeyboardReobserveHint);
|
|
6374
|
+
error.Data["reobserveRequired"] = true;
|
|
6375
|
+
throw error;
|
|
6376
|
+
}
|
|
5581
6377
|
static readonly KickoffFocus _kickoffFocus = new KickoffFocus();
|
|
5582
6378
|
public static void ResetKickoffFocus() { _kickoffFocus.Reset(); }
|
|
5583
6379
|
public static long RecoveryTarget(out uint pid)
|
|
@@ -5587,7 +6383,9 @@ namespace DshComputerUse.Worker
|
|
|
5587
6383
|
long target = _kickoffFocus.Target;
|
|
5588
6384
|
string problem = _kickoffFocus.CaptureProblem(true, false, false,
|
|
5589
6385
|
Native.IsWindow(new IntPtr(target)), foreground.ToInt64(), pid, false);
|
|
5590
|
-
|
|
6386
|
+
if (problem == null) return target;
|
|
6387
|
+
// Same-family foreground (a dialog of this app) is the target now.
|
|
6388
|
+
return target != 0 && SameInputFamily(new IntPtr(target), foreground) ? foreground.ToInt64() : 0;
|
|
5591
6389
|
}
|
|
5592
6390
|
static void ValidateKickoffCapture(Dictionary<string, object> a)
|
|
5593
6391
|
{
|
|
@@ -5597,6 +6395,9 @@ namespace DshComputerUse.Worker
|
|
|
5597
6395
|
IntPtr fg = TopLevel(Native.GetForegroundWindow());
|
|
5598
6396
|
string problem = _kickoffFocus.CaptureProblem(Panic.CycleLit, Panic.Stopped, Panic.Exited,
|
|
5599
6397
|
target != IntPtr.Zero && Native.IsWindow(target), fg.ToInt64(), PidOf(fg), IsHostWindow(fg));
|
|
6398
|
+
// A dialog of the same app is the target now (its hwnd replaced the one we confirmed):
|
|
6399
|
+
// refusing there blocked the very screenshot that shows what the dialog is asking.
|
|
6400
|
+
if (problem != null && target != IntPtr.Zero && SameInputFamily(target, fg)) return;
|
|
5600
6401
|
if (problem != null)
|
|
5601
6402
|
{
|
|
5602
6403
|
// NAME THE ACTUAL FOREGROUND (test report 2026-11-04): "identify the intended
|
|
@@ -5918,6 +6719,39 @@ namespace DshComputerUse.Worker
|
|
|
5918
6719
|
return IsHostPid(unchecked((int)PidOf(h)));
|
|
5919
6720
|
}
|
|
5920
6721
|
|
|
6722
|
+
/// True when `child` is `ancestor` itself or a descendant process of it (bounded walk).
|
|
6723
|
+
/// This is the ONLY relationship that makes a covered click legitimate: the target app's
|
|
6724
|
+
/// own embedded renderer (e.g. WeChatAppEx under Weixin.exe). An unrelated window that
|
|
6725
|
+
/// happens to be on top must never receive the input.
|
|
6726
|
+
public static bool ProcessDescendsFrom(int child, int ancestor)
|
|
6727
|
+
{
|
|
6728
|
+
if (child <= 0 || ancestor <= 0) return false;
|
|
6729
|
+
if (child == ancestor) return true;
|
|
6730
|
+
IntPtr snapshot = Native.CreateToolhelp32Snapshot(Native.TH32CS_SNAPPROCESS, 0);
|
|
6731
|
+
if (snapshot == IntPtr.Zero || snapshot == new IntPtr(-1)) return false;
|
|
6732
|
+
try
|
|
6733
|
+
{
|
|
6734
|
+
uint size = (uint)Marshal.SizeOf(typeof(Native.PROCESSENTRY32W));
|
|
6735
|
+
int pid = child;
|
|
6736
|
+
for (int hop = 0; hop < 16 && pid > 0; hop++)
|
|
6737
|
+
{
|
|
6738
|
+
var entry = new Native.PROCESSENTRY32W();
|
|
6739
|
+
entry.dwSize = size;
|
|
6740
|
+
if (!Native.Process32FirstW(snapshot, ref entry)) return false;
|
|
6741
|
+
int parent = 0;
|
|
6742
|
+
do
|
|
6743
|
+
{
|
|
6744
|
+
if (unchecked((int)entry.th32ProcessID) == pid) { parent = unchecked((int)entry.th32ParentProcessID); break; }
|
|
6745
|
+
} while (Native.Process32NextW(snapshot, ref entry));
|
|
6746
|
+
if (parent <= 0) return false;
|
|
6747
|
+
if (parent == ancestor) return true;
|
|
6748
|
+
pid = parent;
|
|
6749
|
+
}
|
|
6750
|
+
return false;
|
|
6751
|
+
}
|
|
6752
|
+
finally { try { Native.CloseHandle(snapshot); } catch { } }
|
|
6753
|
+
}
|
|
6754
|
+
|
|
5921
6755
|
/// Did the host identity reach us from the launcher? The keyboard half of the human-presence
|
|
5922
6756
|
/// monitor is gated on it (see Panic.MonitorHuman / HookCallback): without it we cannot
|
|
5923
6757
|
/// exempt "the human is talking to the agent in the chat", and braking on every chat message
|
|
@@ -6170,6 +7004,45 @@ namespace DshComputerUse.Worker
|
|
|
6170
7004
|
catch { return false; }
|
|
6171
7005
|
}
|
|
6172
7006
|
|
|
7007
|
+
static int ProcessIntegrityLevel(uint pid)
|
|
7008
|
+
{
|
|
7009
|
+
if (pid == 0) return -1;
|
|
7010
|
+
IntPtr proc = IntPtr.Zero, token = IntPtr.Zero, buffer = IntPtr.Zero;
|
|
7011
|
+
try {
|
|
7012
|
+
proc = Native.OpenProcess(Native.PROCESS_QUERY_LIMITED_INFORMATION, false, pid);
|
|
7013
|
+
if (proc == IntPtr.Zero || !Native.OpenProcessToken(proc, Native.TOKEN_QUERY, out token)) return -1;
|
|
7014
|
+
uint needed;
|
|
7015
|
+
Native.GetTokenInformationBuffer(token, 25, IntPtr.Zero, 0, out needed); // TokenIntegrityLevel
|
|
7016
|
+
if (needed < IntPtr.Size || needed > 4096) return -1;
|
|
7017
|
+
buffer = Marshal.AllocHGlobal((int)needed);
|
|
7018
|
+
if (!Native.GetTokenInformationBuffer(token, 25, buffer, needed, out needed)) return -1;
|
|
7019
|
+
IntPtr sid = Marshal.ReadIntPtr(buffer);
|
|
7020
|
+
if (sid == IntPtr.Zero) return -1;
|
|
7021
|
+
IntPtr countPointer = Native.GetSidSubAuthorityCount(sid);
|
|
7022
|
+
if (countPointer == IntPtr.Zero) return -1;
|
|
7023
|
+
byte count = Marshal.ReadByte(countPointer);
|
|
7024
|
+
if (count == 0 || count > 15) return -1;
|
|
7025
|
+
IntPtr rid = Native.GetSidSubAuthority(sid, (uint)(count - 1));
|
|
7026
|
+
return rid == IntPtr.Zero ? -1 : Marshal.ReadInt32(rid);
|
|
7027
|
+
} catch { return -1; }
|
|
7028
|
+
finally {
|
|
7029
|
+
if (buffer != IntPtr.Zero) Marshal.FreeHGlobal(buffer);
|
|
7030
|
+
if (token != IntPtr.Zero) Native.CloseHandle(token);
|
|
7031
|
+
if (proc != IntPtr.Zero) Native.CloseHandle(proc);
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
static bool AsyncKeyIntegrityAllows(int self, int foreground)
|
|
7035
|
+
{ return self >= 0 && foreground >= 0 && foreground <= self; }
|
|
7036
|
+
internal static bool CanObserveAsyncKeys(out IntPtr foreground)
|
|
7037
|
+
{
|
|
7038
|
+
foreground = Native.GetForegroundWindow();
|
|
7039
|
+
if (foreground == IntPtr.Zero) return false;
|
|
7040
|
+
uint pid = PidOf(foreground);
|
|
7041
|
+
// Unknown is NOT unelevated. GetAsyncKeyState reports zero for UIPI
|
|
7042
|
+
// failures too, so only known same/lower integrity is usable evidence.
|
|
7043
|
+
return AsyncKeyIntegrityAllows(ProcessIntegrityLevel((uint)Process.GetCurrentProcess().Id), ProcessIntegrityLevel(pid)) &&
|
|
7044
|
+
Native.GetForegroundWindow() == foreground && PidOf(foreground) == pid;
|
|
7045
|
+
}
|
|
6173
7046
|
static bool ProcessElevated(uint pid)
|
|
6174
7047
|
{
|
|
6175
7048
|
if (pid == 0) return false;
|
|
@@ -6309,6 +7182,7 @@ namespace DshComputerUse.Worker
|
|
|
6309
7182
|
/// reach because of integrity levels.
|
|
6310
7183
|
static Dictionary<string, object> FocusReport(Dictionary<string, object> a, bool refocus)
|
|
6311
7184
|
{
|
|
7185
|
+
if (refocus) CheckKeyboardReobserve(); // explicit old expectHwnd is not fresh authorization
|
|
6312
7186
|
IntPtr want = IntPtr.Zero;
|
|
6313
7187
|
if (a != null && a.ContainsKey("expectHwnd")) want = new IntPtr(GetI(a, "expectHwnd", 0));
|
|
6314
7188
|
else if (a != null && a.ContainsKey("hwnd")) want = new IntPtr(GetI(a, "hwnd", 0));
|
|
@@ -6352,6 +7226,113 @@ namespace DshComputerUse.Worker
|
|
|
6352
7226
|
|
|
6353
7227
|
|
|
6354
7228
|
|
|
7229
|
+
// ---------- focus: put the keyboard somewhere, and PROVE it before typing ----------
|
|
7230
|
+
// Typing follows the focused CONTROL, not the foreground window: activating Word's Find
|
|
7231
|
+
// dialog and typing still landed in the document (measured 2026-11-07 — "abc" was inserted
|
|
7232
|
+
// into the .doc while the field stayed empty). This op clicks the field, reports where focus
|
|
7233
|
+
// actually went (UIA focused element), and whether a caret is provable there, so a caller
|
|
7234
|
+
// can refuse to type into an unproven field instead of discovering it in the file.
|
|
7235
|
+
static Dictionary<string, object> FocusedElementReport()
|
|
7236
|
+
{
|
|
7237
|
+
try
|
|
7238
|
+
{
|
|
7239
|
+
AutomationElement fe = AutomationElement.FocusedElement;
|
|
7240
|
+
if (fe == null) return Dict("available", false);
|
|
7241
|
+
string type = null, name = null, id = null;
|
|
7242
|
+
try { type = fe.Current.ControlType.ProgrammaticName; } catch { }
|
|
7243
|
+
try { name = fe.Current.Name; } catch { }
|
|
7244
|
+
try { id = fe.Current.AutomationId; } catch { }
|
|
7245
|
+
return Dict("available", true, "controlType", type, "name", Short(name, 48), "automationId", id);
|
|
7246
|
+
}
|
|
7247
|
+
catch (Exception e) { return Dict("available", false, "error", e.GetType().Name); }
|
|
7248
|
+
}
|
|
7249
|
+
|
|
7250
|
+
static Dictionary<string, object> CaretReport()
|
|
7251
|
+
{
|
|
7252
|
+
try
|
|
7253
|
+
{
|
|
7254
|
+
Native.GUITHREADINFO info = new Native.GUITHREADINFO();
|
|
7255
|
+
info.cbSize = Marshal.SizeOf(typeof(Native.GUITHREADINFO));
|
|
7256
|
+
IntPtr fg = Native.GetForegroundWindow();
|
|
7257
|
+
uint pid;
|
|
7258
|
+
uint thread = Native.GetWindowThreadProcessId(fg, out pid);
|
|
7259
|
+
if (thread == 0 || !Native.GetGUIThreadInfo(thread, ref info) || info.hwndCaret == IntPtr.Zero)
|
|
7260
|
+
return Dict("available", false);
|
|
7261
|
+
POINT p = new POINT();
|
|
7262
|
+
p.X = info.rcCaret.Left; p.Y = info.rcCaret.Top;
|
|
7263
|
+
Native.ClientToScreen(info.hwndCaret, ref p);
|
|
7264
|
+
return Dict("available", true, "x", p.X, "y", p.Y,
|
|
7265
|
+
"height", Math.Max(1, info.rcCaret.Bottom - info.rcCaret.Top));
|
|
7266
|
+
}
|
|
7267
|
+
catch (Exception e) { return Dict("available", false, "error", e.GetType().Name); }
|
|
7268
|
+
}
|
|
7269
|
+
|
|
7270
|
+
static Dictionary<string, object> FocusField(Dictionary<string, object> a)
|
|
7271
|
+
{
|
|
7272
|
+
int x = GetI(a, "x", int.MinValue), y = GetI(a, "y", int.MinValue);
|
|
7273
|
+
if (x == int.MinValue || y == int.MinValue)
|
|
7274
|
+
throw new Exception("FOCUS_POINT_REQUIRED: focus needs the field's x,y in screen pixels; nothing was clicked");
|
|
7275
|
+
IntPtr want = a.ContainsKey("hwnd") && GetI(a, "hwnd", 0) != 0 ? new IntPtr(GetI(a, "hwnd", 0)) : IntPtr.Zero;
|
|
7276
|
+
AgentActing(a, "focusField");
|
|
7277
|
+
MarkMutation();
|
|
7278
|
+
if (want != IntPtr.Zero) { RefocusWindow(want, true); Thread.Sleep(60); }
|
|
7279
|
+
AnimatedMove(x, y, GetI(a, "dur", 140));
|
|
7280
|
+
Thread.Sleep(GetI(a, "settleMs", 90));
|
|
7281
|
+
IntPtr under = InputPointReceiver();
|
|
7282
|
+
IntPtr target = want != IntPtr.Zero ? want : TopLevel(under);
|
|
7283
|
+
if (!SameInputFamily(target, under))
|
|
7284
|
+
throw new Exception("precondition failed (COVERED): the pixel belongs to \"" + Short(WinTitleOf(under), 40) + "\" (pid " +
|
|
7285
|
+
PidOf(under) + "), not to the window you aimed at. NOTHING was clicked.");
|
|
7286
|
+
using (GuardReceiver(under, InputWindowPid(under), false))
|
|
7287
|
+
{
|
|
7288
|
+
SendMouse(Native.MOUSEEVENTF_LEFTDOWN, 0, 0, 0);
|
|
7289
|
+
Thread.Sleep(30);
|
|
7290
|
+
SendMouse(Native.MOUSEEVENTF_LEFTUP, 0, 0, 0);
|
|
7291
|
+
}
|
|
7292
|
+
// The click moved focus inside this app: confirm the foreground as the observation
|
|
7293
|
+
// target, or the capture that must prove the field would be refused as unconfirmed.
|
|
7294
|
+
IntPtr fgTop = TopLevel(Native.GetForegroundWindow());
|
|
7295
|
+
if (fgTop != IntPtr.Zero)
|
|
7296
|
+
{
|
|
7297
|
+
SetSticky(fgTop);
|
|
7298
|
+
_kickoffFocus.Confirm(fgTop.ToInt64(), PidOf(fgTop));
|
|
7299
|
+
}
|
|
7300
|
+
EndActuationForReadback();
|
|
7301
|
+
Thread.Sleep(GetI(a, "readDelayMs", 180));
|
|
7302
|
+
Dictionary<string, object> focused = FocusedElementReport();
|
|
7303
|
+
Dictionary<string, object> caret = CaretReport();
|
|
7304
|
+
object rawType;
|
|
7305
|
+
string controlType = focused.TryGetValue("controlType", out rawType) ? Convert.ToString(rawType) : null;
|
|
7306
|
+
bool textTarget = controlType != null &&
|
|
7307
|
+
(controlType.IndexOf("Edit", StringComparison.OrdinalIgnoreCase) >= 0 ||
|
|
7308
|
+
controlType.IndexOf("Document", StringComparison.OrdinalIgnoreCase) >= 0);
|
|
7309
|
+
bool caretVisible = caret.ContainsKey("available") && Convert.ToBoolean(caret["available"]);
|
|
7310
|
+
// UNCERTAIN CASE: the OS reports no caret (browser content, some editors). Fall back to
|
|
7311
|
+
// the one property a caret always has — it blinks: two tight crops 260 ms apart differ.
|
|
7312
|
+
bool blink = false;
|
|
7313
|
+
if (!caretVisible && GetB(a, "verifyCaret", true))
|
|
7314
|
+
{
|
|
7315
|
+
int bx = Math.Max(0, x - 30), by = Math.Max(0, y - 16);
|
|
7316
|
+
byte[] first = FrameSignatureRect(bx, by, 60, 32, 2);
|
|
7317
|
+
Thread.Sleep(260);
|
|
7318
|
+
byte[] second = FrameSignatureRect(bx, by, 60, 32, 2);
|
|
7319
|
+
blink = first != null && second != null && first.Length == second.Length && SigDiffPct(first, second, 8) >= 0.05;
|
|
7320
|
+
}
|
|
7321
|
+
bool verified = textTarget && (caretVisible || blink);
|
|
7322
|
+
return Dict(
|
|
7323
|
+
"clicked", Dict("x", x, "y", y),
|
|
7324
|
+
"receiver", under == IntPtr.Zero ? null : (object)(Short(WinTitleOf(under), 40) + " (pid " + PidOf(under) + ")"),
|
|
7325
|
+
"focused", focused,
|
|
7326
|
+
"textTarget", textTarget,
|
|
7327
|
+
"caret", caret,
|
|
7328
|
+
"caretVerified", caretVisible,
|
|
7329
|
+
"caretBlink", blink,
|
|
7330
|
+
"verified", verified,
|
|
7331
|
+
"reason", verified ? null : (textTarget
|
|
7332
|
+
? "focus reached a text control but no caret is provable; capture the field, click it again, and re-check before typing"
|
|
7333
|
+
: "focus landed on a non-text control (" + (controlType ?? "unknown") + "); click a field before typing"));
|
|
7334
|
+
}
|
|
7335
|
+
|
|
6355
7336
|
static Dictionary<string, object> WindowOp(Dictionary<string, object> a)
|
|
6356
7337
|
{
|
|
6357
7338
|
IntPtr h = FindHwnd(a);
|
|
@@ -6386,6 +7367,43 @@ namespace DshComputerUse.Worker
|
|
|
6386
7367
|
}
|
|
6387
7368
|
|
|
6388
7369
|
// ---------- UIA ----------
|
|
7370
|
+
static void AddUiGeometry(Dictionary<string, object> d, System.Windows.Rect? bounds,
|
|
7371
|
+
bool? offscreen, Rectangle virtualScreen)
|
|
7372
|
+
{
|
|
7373
|
+
bool valid = false;
|
|
7374
|
+
if (bounds.HasValue) {
|
|
7375
|
+
System.Windows.Rect r = bounds.Value;
|
|
7376
|
+
// Empty UIA rectangles contain infinities. Casting these to int
|
|
7377
|
+
// manufactured INT32_MIN coordinates that looked like real layout.
|
|
7378
|
+
valid = !r.IsEmpty && r.Width > 0 && r.Height > 0 &&
|
|
7379
|
+
!Double.IsNaN(r.X) && !Double.IsNaN(r.Y) &&
|
|
7380
|
+
!Double.IsNaN(r.Width) && !Double.IsNaN(r.Height) &&
|
|
7381
|
+
r.X > Int32.MinValue && r.Y > Int32.MinValue &&
|
|
7382
|
+
r.Right < Int32.MaxValue && r.Bottom < Int32.MaxValue &&
|
|
7383
|
+
r.Width < Int32.MaxValue && r.Height < Int32.MaxValue;
|
|
7384
|
+
if (valid) {
|
|
7385
|
+
long left = (long)Math.Floor(r.Left), top = (long)Math.Floor(r.Top);
|
|
7386
|
+
long right = (long)Math.Ceiling(r.Right), bottom = (long)Math.Ceiling(r.Bottom);
|
|
7387
|
+
valid = left > Int32.MinValue && top > Int32.MinValue &&
|
|
7388
|
+
right - left <= Int32.MaxValue && bottom - top <= Int32.MaxValue;
|
|
7389
|
+
if (valid) {
|
|
7390
|
+
d["rect"] = Dict("x", (int)left, "y", (int)top, "width", (int)(right - left), "height", (int)(bottom - top));
|
|
7391
|
+
d["intersectsVirtualScreen"] = r.Right > virtualScreen.Left && r.Left < virtualScreen.Right &&
|
|
7392
|
+
r.Bottom > virtualScreen.Top && r.Top < virtualScreen.Bottom;
|
|
7393
|
+
}
|
|
7394
|
+
}
|
|
7395
|
+
}
|
|
7396
|
+
d["rectValid"] = valid;
|
|
7397
|
+
d["isOffscreen"] = offscreen.HasValue ? (object)offscreen.Value : null;
|
|
7398
|
+
if (!valid) {
|
|
7399
|
+
d["rect"] = null;
|
|
7400
|
+
d["intersectsVirtualScreen"] = null;
|
|
7401
|
+
d["rectInvalidReason"] = bounds.HasValue ? "empty-or-invalid-provider-bounds" : "provider-bounds-unavailable";
|
|
7402
|
+
}
|
|
7403
|
+
// Neither a valid rectangle nor IsOffscreen=false proves that this
|
|
7404
|
+
// element owns a pixel; use the point probe for occlusion/hit testing.
|
|
7405
|
+
}
|
|
7406
|
+
|
|
6389
7407
|
static Dictionary<string, object> ElemDict(AutomationElement e)
|
|
6390
7408
|
{
|
|
6391
7409
|
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
@@ -6395,10 +7413,15 @@ namespace DshComputerUse.Worker
|
|
|
6395
7413
|
string ctype = "";
|
|
6396
7414
|
try { if (e.Current.ControlType != null) ctype = e.Current.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
|
|
6397
7415
|
catch { }
|
|
6398
|
-
System.Windows.Rect r = e.Current.BoundingRectangle;
|
|
6399
7416
|
d["name"] = name;
|
|
6400
7417
|
d["role"] = ctype;
|
|
6401
|
-
|
|
7418
|
+
System.Windows.Rect? bounds = null;
|
|
7419
|
+
bool? offscreen = null;
|
|
7420
|
+
try { bounds = e.Current.BoundingRectangle; } catch { }
|
|
7421
|
+
try { offscreen = e.Current.IsOffscreen; } catch { }
|
|
7422
|
+
AddUiGeometry(d, bounds, offscreen, new Rectangle(
|
|
7423
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
7424
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
6402
7425
|
try { d["automationId"] = e.Current.AutomationId; } catch { }
|
|
6403
7426
|
try { d["className"] = e.Current.ClassName; } catch { }
|
|
6404
7427
|
try { d["runtimeId"] = String.Join(".", e.GetRuntimeId()); d["pid"] = e.Current.ProcessId; } catch { }
|
|
@@ -6424,61 +7447,205 @@ namespace DshComputerUse.Worker
|
|
|
6424
7447
|
return d;
|
|
6425
7448
|
}
|
|
6426
7449
|
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
7450
|
+
// >>>UIA-QUERY-BUDGET<<<
|
|
7451
|
+
// Cooperative limits between provider calls. A synchronous UIA/COM call
|
|
7452
|
+
// already in progress is NOT interruptible by this traversal budget.
|
|
7453
|
+
internal sealed class UiBudgetStop : Exception { }
|
|
7454
|
+
internal sealed class UiReadBudget
|
|
7455
|
+
{
|
|
7456
|
+
readonly Func<long> clock;
|
|
7457
|
+
readonly long started;
|
|
7458
|
+
readonly List<string> reasons = new List<string>();
|
|
7459
|
+
public readonly int BudgetMs;
|
|
7460
|
+
public int Calls, Scanned;
|
|
7461
|
+
public long MaxCallMs;
|
|
7462
|
+
public UiReadBudget(int budgetMs, Func<long> now)
|
|
7463
|
+
{ BudgetMs = Math.Max(100, Math.Min(5000, budgetMs)); clock = now; started = now(); }
|
|
7464
|
+
public long ElapsedMs { get { return Math.Max(0, clock() - started); } }
|
|
7465
|
+
public bool Truncated { get { return reasons.Count > 0; } }
|
|
7466
|
+
public void Mark(string reason) { if (!reasons.Contains(reason)) reasons.Add(reason); }
|
|
7467
|
+
public bool Continue()
|
|
7468
|
+
{ if (ElapsedMs < BudgetMs) return true; Mark("deadline"); return false; }
|
|
7469
|
+
public T Read<T>(Func<T> read)
|
|
7470
|
+
{
|
|
7471
|
+
if (!Continue()) throw new UiBudgetStop();
|
|
7472
|
+
long before = clock(); Calls++;
|
|
7473
|
+
try {
|
|
7474
|
+
T value = read();
|
|
7475
|
+
if (!Continue()) throw new UiBudgetStop();
|
|
7476
|
+
return value;
|
|
7477
|
+
} finally { MaxCallMs = Math.Max(MaxCallMs, Math.Max(0, clock() - before)); }
|
|
7478
|
+
}
|
|
7479
|
+
public Dictionary<string, object> Report(int maxNodes, int depth)
|
|
7480
|
+
{
|
|
7481
|
+
return Dict("budgetMs", BudgetMs, "elapsedMs", ElapsedMs, "maxNodes", maxNodes, "depth", depth,
|
|
7482
|
+
"scanned", Scanned, "providerCalls", Calls, "maxCallMs", MaxCallMs,
|
|
7483
|
+
"truncatedReasons", reasons.ToArray(), "singleCallInterruptible", false,
|
|
7484
|
+
"note", "Soft traversal budget checked between provider calls; one synchronous UIA/COM call can exceed it. A partial result cannot prove absence or uniqueness.");
|
|
7485
|
+
}
|
|
7486
|
+
}
|
|
7487
|
+
static long UiClockMs()
|
|
7488
|
+
{ return (long)(Stopwatch.GetTimestamp() * (1000.0 / Stopwatch.Frequency)); }
|
|
7489
|
+
static UiReadBudget NewUiBudget(Dictionary<string, object> a)
|
|
7490
|
+
{ return new UiReadBudget(GetI(a, "budgetMs", 1200), UiClockMs); }
|
|
7491
|
+
|
|
7492
|
+
internal static void ValidateUiQueryScope(bool scoped, bool filtered, bool anchored, int depth, int maxNodes)
|
|
7493
|
+
{
|
|
7494
|
+
if (filtered && !scoped)
|
|
7495
|
+
throw new Exception("UIA_SCOPE_REQUIRED: filtered UIA queries require hwnd or an unambiguous window title; use computer_state to identify the window, or computer_uia at for a point probe.");
|
|
7496
|
+
if (!anchored && (depth > 6 || maxNodes > 300))
|
|
7497
|
+
throw new Exception("UIA_ANCHOR_REQUIRED: broad UIA traversal requires an observed anchor point in the requested window; use computer_uia at first, then anchor with depth/maxNodes limits. An exact id does not avoid traversal.");
|
|
7498
|
+
}
|
|
7499
|
+
internal static bool UiAnchorOwned(long requestedTop, uint requestedPid, long actualTop, uint actualPid)
|
|
7500
|
+
{ return requestedTop != 0 && requestedPid != 0 && requestedTop == actualTop && requestedPid == actualPid; }
|
|
7501
|
+
static AutomationElement UiQueryRoot(Dictionary<string, object> a, IntPtr root, UiReadBudget budget)
|
|
7502
|
+
{
|
|
7503
|
+
string anchor = GetS(a, "anchor", null);
|
|
7504
|
+
if (String.IsNullOrWhiteSpace(anchor)) return budget.Read(delegate { return AutomationElement.FromHandle(root); });
|
|
7505
|
+
string[] xy = anchor.Split(','); int x, y;
|
|
7506
|
+
if (xy.Length != 2 || !Int32.TryParse(xy[0].Trim(), out x) || !Int32.TryParse(xy[1].Trim(), out y))
|
|
7507
|
+
throw new Exception("UIA_ANCHOR_INVALID: anchor must be physical screen x,y");
|
|
7508
|
+
POINT point; point.X = x; point.Y = y;
|
|
7509
|
+
uint pid = PidOf(root); IntPtr expectedTop = TopLevel(root), hit = Native.WindowFromPoint(point);
|
|
7510
|
+
if (!Native.IsWindow(root) || !UiAnchorOwned(expectedTop.ToInt64(), pid, TopLevel(hit).ToInt64(), PidOf(hit)))
|
|
7511
|
+
throw new Exception("UIA_ANCHOR_MISMATCH: anchor is not currently owned by the requested window and process; no provider query was started");
|
|
7512
|
+
AutomationElement element = budget.Read(delegate { return AutomationElement.FromPoint(new System.Windows.Point(x, y)); });
|
|
7513
|
+
IntPtr afterHit = Native.WindowFromPoint(point);
|
|
7514
|
+
if (element == null || budget.Read(delegate { return element.Current.ProcessId; }) != (int)pid ||
|
|
7515
|
+
PidOf(root) != pid || !UiAnchorOwned(expectedTop.ToInt64(), pid, TopLevel(afterHit).ToInt64(), PidOf(afterHit)))
|
|
7516
|
+
throw new Exception("UIA_ANCHOR_MISMATCH: provider hit or window changed during the point probe; repeat the observation");
|
|
7517
|
+
return element;
|
|
7518
|
+
}
|
|
7519
|
+
|
|
7520
|
+
// Generic only so deterministic fake trees exercise the identical walk.
|
|
7521
|
+
// Child/sibling navigation is incremental: no FindAll collection can be
|
|
7522
|
+
// built before the client-side node/depth/deadline checks run.
|
|
7523
|
+
internal static List<T> UiWalkBounded<T>(T start, int depth, int maxNodes, int limit, UiReadBudget budget,
|
|
7524
|
+
Func<T, bool> matches, Func<T, T> firstChild, Func<T, T> nextSibling) where T : class
|
|
7525
|
+
{
|
|
7526
|
+
var found = new List<T>(); var queue = new Queue<KeyValuePair<T, int>>();
|
|
7527
|
+
if (start == null) return found;
|
|
7528
|
+
// A queue entry is a parent whose direct children form the next BFS
|
|
7529
|
+
// segment. Visit each child before asking for its next sibling: a
|
|
7530
|
+
// wide provider can no longer consume the budget before any match
|
|
7531
|
+
// in that sibling list has been inspected/materialized.
|
|
7532
|
+
queue.Enqueue(new KeyValuePair<T, int>(start, -1));
|
|
7533
|
+
while (queue.Count > 0 && budget.Scanned < maxNodes && budget.Continue())
|
|
7534
|
+
{
|
|
7535
|
+
var segment = queue.Dequeue(); int level = segment.Value + 1;
|
|
7536
|
+
try
|
|
7537
|
+
{
|
|
7538
|
+
T node = segment.Value < 0 ? segment.Key : budget.Read(delegate { return firstChild(segment.Key); });
|
|
7539
|
+
while (node != null && budget.Continue())
|
|
7540
|
+
{
|
|
7541
|
+
if (budget.Scanned >= maxNodes) { budget.Mark("max-nodes"); return found; }
|
|
7542
|
+
budget.Scanned++;
|
|
7543
|
+
try {
|
|
7544
|
+
if (matches(node)) {
|
|
7545
|
+
found.Add(node);
|
|
7546
|
+
if (found.Count >= limit) { budget.Mark("match-limit"); return found; }
|
|
7547
|
+
}
|
|
7548
|
+
if (!budget.Continue()) return found;
|
|
7549
|
+
if (level < depth) queue.Enqueue(new KeyValuePair<T, int>(node, level));
|
|
7550
|
+
else if (budget.Read(delegate { return firstChild(node); }) != null) budget.Mark("depth");
|
|
7551
|
+
} catch (UiBudgetStop) { return found; }
|
|
7552
|
+
catch { budget.Mark("provider-error"); }
|
|
7553
|
+
if (segment.Value < 0) break;
|
|
7554
|
+
if (budget.Scanned >= maxNodes) { budget.Mark("max-nodes"); return found; }
|
|
7555
|
+
T previous = node;
|
|
7556
|
+
node = budget.Read(delegate { return nextSibling(previous); });
|
|
7557
|
+
}
|
|
7558
|
+
}
|
|
7559
|
+
catch (UiBudgetStop) { break; }
|
|
7560
|
+
catch { budget.Mark("provider-error"); }
|
|
7561
|
+
}
|
|
7562
|
+
if (queue.Count > 0) budget.Mark(budget.Scanned >= maxNodes ? "max-nodes" : "pending-nodes");
|
|
7563
|
+
return found;
|
|
7564
|
+
}
|
|
7565
|
+
// <<<UIA-QUERY-BUDGET<<<
|
|
7566
|
+
|
|
7567
|
+
static Dictionary<string, object> ElemDictBudgeted(AutomationElement e, UiReadBudget budget)
|
|
7568
|
+
{
|
|
7569
|
+
var d = new Dictionary<string, object>();
|
|
7570
|
+
string name = "", role = ""; System.Windows.Rect? bounds = null; bool? offscreen = null;
|
|
7571
|
+
try { name = budget.Read(delegate { return e.Current.Name; }) ?? ""; } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7572
|
+
try { ControlType ct = budget.Read(delegate { return e.Current.ControlType; }); if (ct != null) role = ct.ProgrammaticName.Replace("ControlType.", ""); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7573
|
+
d["name"] = name.Length > 160 ? name.Substring(0, 160) : name; d["role"] = role;
|
|
7574
|
+
try { bounds = budget.Read(delegate { return e.Current.BoundingRectangle; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7575
|
+
try { offscreen = budget.Read(delegate { return e.Current.IsOffscreen; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7576
|
+
AddUiGeometry(d, bounds, offscreen, new Rectangle(Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
7577
|
+
try { d["automationId"] = budget.Read(delegate { return e.Current.AutomationId; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7578
|
+
try { d["className"] = budget.Read(delegate { return e.Current.ClassName; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7579
|
+
try { d["runtimeId"] = String.Join(".", budget.Read(delegate { return e.GetRuntimeId(); })); d["pid"] = budget.Read(delegate { return e.Current.ProcessId; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7580
|
+
try { d["isPassword"] = budget.Read(delegate { return e.Current.IsPassword; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7581
|
+
try {
|
|
7582
|
+
var patterns = new List<string>();
|
|
7583
|
+
foreach (AutomationPattern pattern in budget.Read(delegate { return e.GetSupportedPatterns(); })) {
|
|
7584
|
+
string text = pattern.ProgrammaticName.Replace("PatternIdentifiers", "").Replace("Pattern", "").TrimEnd('.', ' ');
|
|
7585
|
+
if (text.Length > 0) patterns.Add(text);
|
|
7586
|
+
}
|
|
7587
|
+
d["patterns"] = patterns;
|
|
7588
|
+
} catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7589
|
+
try {
|
|
7590
|
+
object value = budget.Read(delegate { object p; return e.TryGetCurrentPattern(ValuePattern.Pattern, out p) ? p : null; });
|
|
7591
|
+
if (value != null) d["value"] = budget.Read(delegate { return ((ValuePattern)value).Current.Value; });
|
|
7592
|
+
} catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7593
|
+
return d;
|
|
7594
|
+
}
|
|
7595
|
+
|
|
7596
|
+
static Dictionary<string, object> Uia(Dictionary<string, object> a)
|
|
7597
|
+
{
|
|
7598
|
+
IntPtr root = FindHwnd(a);
|
|
6431
7599
|
int maxDepth = Math.Max(1, Math.Min(24, GetI(a, "depth", 6)));
|
|
6432
|
-
int maxNodes = Math.Max(10, Math.Min(1200, GetI(a, "maxNodes",
|
|
6433
|
-
|
|
7600
|
+
int maxNodes = Math.Max(10, Math.Min(1200, GetI(a, "maxNodes", 120)));
|
|
7601
|
+
bool query = GetB(a, "query", false), anchored = !String.IsNullOrWhiteSpace(GetS(a, "anchor", null));
|
|
7602
|
+
ValidateUiQueryScope(root != IntPtr.Zero, query, anchored, maxDepth, maxNodes);
|
|
7603
|
+
if (root == IntPtr.Zero) root = Native.GetForegroundWindow();
|
|
7604
|
+
if (root == IntPtr.Zero) throw new Exception("UIA_SCOPE_REQUIRED: no foreground window; specify hwnd");
|
|
7605
|
+
UiReadBudget budget = NewUiBudget(a);
|
|
7606
|
+
AutomationElement rootEl;
|
|
7607
|
+
try { rootEl = UiQueryRoot(a, root, budget); }
|
|
7608
|
+
catch (UiBudgetStop) { return Dict("flat", new List<object>(), "count", 0, "scanned", 0, "truncated", true, "budget", budget.Report(maxNodes, maxDepth), "hwnd", root.ToInt64(), "pid", PidOf(root)); }
|
|
7609
|
+
if (query)
|
|
6434
7610
|
{
|
|
6435
|
-
int scanned; bool truncated;
|
|
6436
|
-
List<AutomationElement> matches = FindUiMatches(rootEl, GetS(a, "nameContains", null),
|
|
6437
|
-
GetS(a, "automationId", null), GetS(a, "role", null), maxDepth, maxNodes,
|
|
6438
|
-
Math.Max(1, Math.Min(80, GetI(a, "limit", 80))), out scanned, out truncated);
|
|
6439
7611
|
List<object> selected = new List<object>();
|
|
6440
7612
|
string witness = root != IntPtr.Zero && GetB(a, "targets", false) ? UiWindowWitness(root) : null;
|
|
6441
7613
|
var ancestorCache = new Dictionary<string, List<object>>();
|
|
6442
|
-
|
|
6443
|
-
{
|
|
6444
|
-
try {
|
|
6445
|
-
|
|
7614
|
+
AutomationElement evidenceRoot = rootEl;
|
|
7615
|
+
if (witness != null && anchored) {
|
|
7616
|
+
try { evidenceRoot = budget.Read(delegate { return AutomationElement.FromHandle(root); }); }
|
|
7617
|
+
catch (UiBudgetStop) { evidenceRoot = null; }
|
|
7618
|
+
}
|
|
7619
|
+
string nameFilter = GetS(a, "nameContains", null), idFilter = GetS(a, "automationId", null), roleFilter = GetS(a, "role", null);
|
|
7620
|
+
CacheRequest selectorCache = UiSelectorCache();
|
|
7621
|
+
UiWalkBounded(rootEl, maxDepth, maxNodes, Math.Max(1, Math.Min(80, GetI(a, "limit", 80))), budget,
|
|
7622
|
+
delegate(AutomationElement match) {
|
|
7623
|
+
if (!ElementMatchesBudgeted(match, nameFilter, idFilter, roleFilter, budget)) return false;
|
|
7624
|
+
var item = ElemDictBudgeted(match, budget);
|
|
6446
7625
|
if (witness != null) {
|
|
6447
|
-
try {
|
|
7626
|
+
try {
|
|
7627
|
+
if (evidenceRoot == null) throw new UiBudgetStop();
|
|
7628
|
+
item["target"] = RememberUiTarget(match, root, witness, evidenceRoot, ancestorCache, item, budget);
|
|
7629
|
+
}
|
|
7630
|
+
catch (UiBudgetStop) { item["targetUnavailable"] = true; }
|
|
6448
7631
|
catch { item["targetUnavailable"] = true; }
|
|
6449
7632
|
}
|
|
6450
7633
|
selected.Add(item);
|
|
6451
|
-
|
|
6452
|
-
|
|
7634
|
+
return true;
|
|
7635
|
+
},
|
|
7636
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element, selectorCache); },
|
|
7637
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element, selectorCache); });
|
|
6453
7638
|
if (witness != null && witness != UiWindowWitness(root)) throw new Exception("TARGET_STALE: window changed during observation; retry the query");
|
|
6454
|
-
return Dict("flat", selected, "count", selected.Count, "scanned",
|
|
6455
|
-
"hwnd", root.ToInt64(), "pid", root
|
|
7639
|
+
return Dict("flat", selected, "count", selected.Count, "scanned", budget.Scanned, "truncated", budget.Truncated,
|
|
7640
|
+
"hwnd", root.ToInt64(), "pid", PidOf(root), "budget", budget.Report(maxNodes, maxDepth));
|
|
6456
7641
|
}
|
|
6457
7642
|
List<object> flat = new List<object>();
|
|
6458
|
-
|
|
6459
|
-
|
|
6460
|
-
|
|
6461
|
-
|
|
6462
|
-
|
|
6463
|
-
|
|
6464
|
-
KeyValuePair<AutomationElement, int> kv = q.Dequeue();
|
|
6465
|
-
try { flat.Add(ElemDict(kv.Key)); }
|
|
6466
|
-
catch { treeTruncated = true; continue; }
|
|
6467
|
-
if (kv.Value < maxDepth)
|
|
6468
|
-
{
|
|
6469
|
-
try
|
|
6470
|
-
{
|
|
6471
|
-
foreach (AutomationElement c in kv.Key.FindAll(TreeScope.Children, Condition.TrueCondition))
|
|
6472
|
-
{
|
|
6473
|
-
if (flat.Count + q.Count >= maxNodes) { treeTruncated = true; break; }
|
|
6474
|
-
q.Enqueue(new KeyValuePair<AutomationElement, int>(c, kv.Value + 1));
|
|
6475
|
-
}
|
|
6476
|
-
}
|
|
6477
|
-
catch { treeTruncated = true; }
|
|
6478
|
-
}
|
|
6479
|
-
else try { if (kv.Key.FindFirst(TreeScope.Children, Condition.TrueCondition) != null) treeTruncated = true; } catch { treeTruncated = true; }
|
|
6480
|
-
}
|
|
6481
|
-
return Dict("root", ElemDict(rootEl), "flat", flat, "count", flat.Count, "truncated", treeTruncated || q.Count > 0);
|
|
7643
|
+
UiWalkBounded(rootEl, maxDepth, maxNodes, maxNodes + 1, budget,
|
|
7644
|
+
delegate(AutomationElement element) { flat.Add(ElemDictBudgeted(element, budget)); return false; },
|
|
7645
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element); },
|
|
7646
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element); });
|
|
7647
|
+
return Dict("root", flat.Count > 0 ? flat[0] : null, "flat", flat, "count", flat.Count,
|
|
7648
|
+
"truncated", budget.Truncated, "budget", budget.Report(maxNodes, maxDepth), "hwnd", root.ToInt64(), "pid", PidOf(root));
|
|
6482
7649
|
}
|
|
6483
7650
|
|
|
6484
7651
|
// Observed native identities are private to this worker, bounded, and never reconstructed
|
|
@@ -6502,31 +7669,35 @@ namespace DshComputerUse.Worker
|
|
|
6502
7669
|
return Json.Serialize(new object[] { hwnd.ToInt64(), pid, started, WinTitleOf(hwnd), ClassOf(hwnd), rect.Left, rect.Top, rect.Right, rect.Bottom });
|
|
6503
7670
|
}
|
|
6504
7671
|
static string UiText(string text) { return (text ?? "").Normalize(NormalizationForm.FormKC).Trim(); }
|
|
6505
|
-
static
|
|
7672
|
+
static T UiReadMaybe<T>(UiReadBudget budget, Func<T> read)
|
|
7673
|
+
{ return budget == null ? read() : budget.Read(read); }
|
|
7674
|
+
static object UiSemanticPart(AutomationElement element, UiReadBudget budget = null)
|
|
6506
7675
|
{
|
|
6507
|
-
return new object[] {
|
|
7676
|
+
return new object[] { UiReadMaybe(budget, delegate { return element.Current.ControlType; }).ProgrammaticName,
|
|
7677
|
+
UiText(UiReadMaybe(budget, delegate { return element.Current.Name; })),
|
|
7678
|
+
UiReadMaybe(budget, delegate { return element.Current.AutomationId; }), UiReadMaybe(budget, delegate { return element.Current.ClassName; }) };
|
|
6508
7679
|
}
|
|
6509
|
-
static string UiTargetEvidence(AutomationElement element, IntPtr hwnd)
|
|
7680
|
+
static string UiTargetEvidence(AutomationElement element, IntPtr hwnd, UiReadBudget budget = null)
|
|
6510
7681
|
{
|
|
6511
|
-
return UiTargetEvidence(element, AutomationElement.FromHandle(hwnd), new Dictionary<string, List<object>>(), null);
|
|
7682
|
+
return UiTargetEvidence(element, UiReadMaybe(budget, delegate { return AutomationElement.FromHandle(hwnd); }), new Dictionary<string, List<object>>(), null, budget);
|
|
6512
7683
|
}
|
|
6513
|
-
static string UiTargetEvidence(AutomationElement element, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item)
|
|
7684
|
+
static string UiTargetEvidence(AutomationElement element, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item, UiReadBudget budget = null)
|
|
6514
7685
|
{
|
|
6515
7686
|
var ancestors = new List<object>();
|
|
6516
|
-
AutomationElement current = TreeWalker.ControlViewWalker.GetParent(element);
|
|
6517
|
-
string parentId = current == null ? "" : String.Join(".", current.GetRuntimeId());
|
|
7687
|
+
AutomationElement current = UiReadMaybe(budget, delegate { return TreeWalker.ControlViewWalker.GetParent(element); });
|
|
7688
|
+
string parentId = current == null ? "" : String.Join(".", UiReadMaybe(budget, delegate { return current.GetRuntimeId(); }));
|
|
6518
7689
|
if (parentId.Length == 0 || !cache.TryGetValue(parentId, out ancestors)) {
|
|
6519
7690
|
ancestors = new List<object>();
|
|
6520
|
-
bool rooted = Automation.Compare(element, root);
|
|
7691
|
+
bool rooted = UiReadMaybe(budget, delegate { return Automation.Compare(element, root); });
|
|
6521
7692
|
for (int depth = 0; current != null && depth < 24; depth++) {
|
|
6522
|
-
ancestors.Add(UiSemanticPart(current));
|
|
6523
|
-
if (Automation.Compare(current, root)) { rooted = true; break; }
|
|
6524
|
-
current = TreeWalker.ControlViewWalker.GetParent(current);
|
|
7693
|
+
ancestors.Add(UiSemanticPart(current, budget));
|
|
7694
|
+
if (UiReadMaybe(budget, delegate { return Automation.Compare(current, root); })) { rooted = true; break; }
|
|
7695
|
+
current = UiReadMaybe(budget, delegate { return TreeWalker.ControlViewWalker.GetParent(current); });
|
|
6525
7696
|
}
|
|
6526
7697
|
if (!rooted) throw new Exception("TARGET_STALE: control is outside the observed window or ancestry is incomplete");
|
|
6527
7698
|
if (parentId.Length > 0) cache[parentId] = ancestors;
|
|
6528
7699
|
}
|
|
6529
|
-
if (item == null) item = ElemDict(element);
|
|
7700
|
+
if (item == null) item = budget == null ? ElemDict(element) : ElemDictBudgeted(element, budget);
|
|
6530
7701
|
var patterns = new List<string>();
|
|
6531
7702
|
object raw;
|
|
6532
7703
|
if (!item.TryGetValue("patterns", out raw)) throw new Exception("TARGET_STALE: native patterns unavailable");
|
|
@@ -6534,14 +7705,14 @@ namespace DshComputerUse.Worker
|
|
|
6534
7705
|
patterns.Sort(StringComparer.Ordinal);
|
|
6535
7706
|
// AutomationId is a separate identity channel, so a unique semantic replacement can
|
|
6536
7707
|
// still be recognized if a provider regenerates IDs. Ancestor identities remain strict.
|
|
6537
|
-
return Json.Serialize(new object[] { item["role"], UiText(element.Current.Name),
|
|
7708
|
+
return Json.Serialize(new object[] { item["role"], UiText(UiReadMaybe(budget, delegate { return element.Current.Name; })),
|
|
6538
7709
|
item["className"], item["isPassword"], patterns, ancestors });
|
|
6539
7710
|
}
|
|
6540
|
-
static string RememberUiTarget(AutomationElement element, IntPtr hwnd, string witness, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item)
|
|
7711
|
+
static string RememberUiTarget(AutomationElement element, IntPtr hwnd, string witness, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item, UiReadBudget budget = null)
|
|
6541
7712
|
{
|
|
6542
7713
|
string runtimeId = GetS(item, "runtimeId", "");
|
|
6543
7714
|
if (runtimeId.Length == 0) throw new Exception("native control has no runtime identity");
|
|
6544
|
-
string evidence = UiTargetEvidence(element, root, cache, item);
|
|
7715
|
+
string evidence = UiTargetEvidence(element, root, cache, item, budget);
|
|
6545
7716
|
string autoId = GetS(item, "automationId", "");
|
|
6546
7717
|
string key = Json.Serialize(new object[] { witness, runtimeId, evidence, autoId });
|
|
6547
7718
|
string existing; ObservedUiTarget old;
|
|
@@ -6550,7 +7721,7 @@ namespace DshComputerUse.Worker
|
|
|
6550
7721
|
}
|
|
6551
7722
|
string token = "T" + Guid.NewGuid().ToString("N");
|
|
6552
7723
|
var target = new ObservedUiTarget { Token = token, Key = key, Hwnd = hwnd, Witness = witness, RuntimeId = runtimeId,
|
|
6553
|
-
Evidence = evidence, AutoId = autoId, Name = element.Current.Name,
|
|
7724
|
+
Evidence = evidence, AutoId = autoId, Name = UiReadMaybe(budget, delegate { return element.Current.Name; }),
|
|
6554
7725
|
Role = GetS(item, "role", ""), Element = element, At = NowMs() };
|
|
6555
7726
|
UiTargets[token] = target; UiTargetKeys[key] = token; UiTargetOrder.Enqueue(token);
|
|
6556
7727
|
while (UiTargets.Count > 512) {
|
|
@@ -6568,36 +7739,38 @@ namespace DshComputerUse.Worker
|
|
|
6568
7739
|
throw NotDispatched("TARGET_STALE", "TARGET_STALE: window identity, title or bounds changed; query again; no action sent");
|
|
6569
7740
|
return target;
|
|
6570
7741
|
}
|
|
6571
|
-
static AutomationElement ResolveUiTarget(ObservedUiTarget target, bool rebind, int depth, int budget, out string mode, out int seen
|
|
7742
|
+
static AutomationElement ResolveUiTarget(ObservedUiTarget target, bool rebind, int depth, int budget, out string mode, out int seen,
|
|
7743
|
+
UiReadBudget readBudget = null, Dictionary<string, object> scope = null)
|
|
6572
7744
|
{
|
|
7745
|
+
if (readBudget == null) readBudget = new UiReadBudget(1200, UiClockMs);
|
|
6573
7746
|
mode = "runtime-id"; seen = 0;
|
|
6574
7747
|
try {
|
|
6575
|
-
if (String.Join(".", target.Element.GetRuntimeId()) == target.RuntimeId &&
|
|
6576
|
-
target.Element.Current.AutomationId == target.AutoId && UiTargetEvidence(target.Element, target.Hwnd) == target.Evidence)
|
|
7748
|
+
if (String.Join(".", readBudget.Read(delegate { return target.Element.GetRuntimeId(); })) == target.RuntimeId &&
|
|
7749
|
+
readBudget.Read(delegate { return target.Element.Current.AutomationId; }) == target.AutoId && UiTargetEvidence(target.Element, target.Hwnd, readBudget) == target.Evidence)
|
|
6577
7750
|
return target.Element;
|
|
6578
|
-
} catch { }
|
|
7751
|
+
} catch (UiBudgetStop) { throw; } catch { }
|
|
6579
7752
|
if (!rebind) throw new Exception("TARGET_STALE: observed control was replaced or changed; query again or explicitly request rebind; no action sent");
|
|
6580
|
-
var start = AutomationElement.FromHandle(target.Hwnd);
|
|
7753
|
+
var start = scope == null ? readBudget.Read(delegate { return AutomationElement.FromHandle(target.Hwnd); }) : UiQueryRoot(scope, target.Hwnd, readBudget);
|
|
6581
7754
|
bool truncated; int visited;
|
|
6582
7755
|
List<AutomationElement> candidates = new List<AutomationElement>();
|
|
6583
7756
|
if (!String.IsNullOrEmpty(target.AutoId)) {
|
|
6584
|
-
candidates =
|
|
7757
|
+
candidates = FindUiMatchesBudgeted(start, null, target.AutoId, null, depth, budget, budget, readBudget, out visited, out truncated);
|
|
6585
7758
|
seen += visited;
|
|
6586
7759
|
if (truncated) throw new Exception("INCOMPLETE_UIA_SEARCH: cannot prove unique target recovery; no action sent");
|
|
6587
7760
|
if (candidates.Count > 1) throw new Exception("AMBIGUOUS_UIA_TARGET: native identity matches multiple controls; no action sent");
|
|
6588
7761
|
if (candidates.Count == 1) {
|
|
6589
|
-
if (UiTargetEvidence(candidates[0], target.Hwnd) != target.Evidence)
|
|
7762
|
+
if (UiTargetEvidence(candidates[0], target.Hwnd, readBudget) != target.Evidence)
|
|
6590
7763
|
throw new Exception("TARGET_SEMANTICS_CHANGED: native ID now names a different control/context; no action sent");
|
|
6591
7764
|
mode = "native-identifier";
|
|
6592
7765
|
}
|
|
6593
7766
|
}
|
|
6594
7767
|
if (candidates.Count == 0) {
|
|
6595
7768
|
if (String.IsNullOrWhiteSpace(target.Name)) throw new Exception("TARGET_STALE: target has no usable semantic name; no action sent");
|
|
6596
|
-
var matches =
|
|
7769
|
+
var matches = FindUiMatchesBudgeted(start, target.Name, null, target.Role, depth, budget, budget, readBudget, out visited, out truncated);
|
|
6597
7770
|
seen += visited;
|
|
6598
7771
|
if (truncated) throw new Exception("INCOMPLETE_UIA_SEARCH: cannot prove unique semantic recovery; no action sent");
|
|
6599
7772
|
foreach (AutomationElement element in matches)
|
|
6600
|
-
if (UiTargetEvidence(element, target.Hwnd) == target.Evidence) candidates.Add(element);
|
|
7773
|
+
if (UiTargetEvidence(element, target.Hwnd, readBudget) == target.Evidence) candidates.Add(element);
|
|
6601
7774
|
if (candidates.Count > 1) throw new Exception("AMBIGUOUS_UIA_TARGET: semantic recovery matches multiple controls; no action sent");
|
|
6602
7775
|
if (candidates.Count == 0) throw new Exception("TARGET_STALE: no control retains the observed identity/context; no action sent");
|
|
6603
7776
|
mode = "semantic";
|
|
@@ -6615,9 +7788,28 @@ namespace DshComputerUse.Worker
|
|
|
6615
7788
|
// button, expanding a dropdown, filling a field.
|
|
6616
7789
|
static bool ElementMatches(AutomationElement e, string nameFilter, string autoIdFilter, string roleFilter)
|
|
6617
7790
|
{
|
|
6618
|
-
|
|
6619
|
-
|
|
6620
|
-
|
|
7791
|
+
return ElementMatchesBudgeted(e, nameFilter, autoIdFilter, roleFilter, null);
|
|
7792
|
+
}
|
|
7793
|
+
static CacheRequest UiSelectorCache()
|
|
7794
|
+
{
|
|
7795
|
+
// Only the current navigated element, never a cached subtree. This
|
|
7796
|
+
// batches the three selector properties into the navigation call.
|
|
7797
|
+
var cache = new CacheRequest(); cache.TreeScope = TreeScope.Element;
|
|
7798
|
+
cache.AutomationElementMode = AutomationElementMode.Full;
|
|
7799
|
+
cache.Add(AutomationElement.NameProperty); cache.Add(AutomationElement.AutomationIdProperty);
|
|
7800
|
+
cache.Add(AutomationElement.ControlTypeProperty); return cache;
|
|
7801
|
+
}
|
|
7802
|
+
static T UiSelectorRead<T>(UiReadBudget budget, Func<T> cached, Func<T> current)
|
|
7803
|
+
{
|
|
7804
|
+
if (budget != null && !budget.Continue()) throw new UiBudgetStop();
|
|
7805
|
+
try { return cached(); }
|
|
7806
|
+
catch (InvalidOperationException) { return UiReadMaybe(budget, current); }
|
|
7807
|
+
}
|
|
7808
|
+
static bool ElementMatchesBudgeted(AutomationElement e, string nameFilter, string autoIdFilter, string roleFilter, UiReadBudget budget)
|
|
7809
|
+
{
|
|
7810
|
+
if (autoIdFilter != null && !String.Equals(UiSelectorRead(budget, delegate { return e.Cached.AutomationId; }, delegate { return e.Current.AutomationId; }), autoIdFilter, StringComparison.Ordinal)) return false;
|
|
7811
|
+
if (roleFilter != null && !String.Equals(UiSelectorRead(budget, delegate { return e.Cached.ControlType; }, delegate { return e.Current.ControlType; }).ProgrammaticName.Replace("ControlType.", ""), roleFilter, StringComparison.OrdinalIgnoreCase)) return false;
|
|
7812
|
+
if (nameFilter != null && (UiSelectorRead(budget, delegate { return e.Cached.Name; }, delegate { return e.Current.Name; }) ?? "").IndexOf(nameFilter, StringComparison.OrdinalIgnoreCase) < 0) return false;
|
|
6621
7813
|
return true;
|
|
6622
7814
|
}
|
|
6623
7815
|
|
|
@@ -6626,32 +7818,19 @@ namespace DshComputerUse.Worker
|
|
|
6626
7818
|
static List<AutomationElement> FindUiMatches(AutomationElement start, string name, string autoId,
|
|
6627
7819
|
string role, int depth, int maxNodes, int limit, out int seen, out bool truncated)
|
|
6628
7820
|
{
|
|
6629
|
-
|
|
6630
|
-
|
|
6631
|
-
|
|
6632
|
-
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
}
|
|
6643
|
-
if (kv.Value < depth)
|
|
6644
|
-
foreach (AutomationElement child in kv.Key.FindAll(TreeScope.Children, Condition.TrueCondition))
|
|
6645
|
-
{
|
|
6646
|
-
if (seen + q.Count >= maxNodes) { truncated = true; break; }
|
|
6647
|
-
q.Enqueue(new KeyValuePair<AutomationElement, int>(child, kv.Value + 1));
|
|
6648
|
-
}
|
|
6649
|
-
else if (kv.Key.FindFirst(TreeScope.Children, Condition.TrueCondition) != null)
|
|
6650
|
-
truncated = true;
|
|
6651
|
-
}
|
|
6652
|
-
catch { truncated = true; }
|
|
6653
|
-
}
|
|
6654
|
-
if (q.Count > 0) truncated = true;
|
|
7821
|
+
return FindUiMatchesBudgeted(start, name, autoId, role, depth, maxNodes, limit,
|
|
7822
|
+
new UiReadBudget(1200, UiClockMs), out seen, out truncated);
|
|
7823
|
+
}
|
|
7824
|
+
static List<AutomationElement> FindUiMatchesBudgeted(AutomationElement start, string name, string autoId,
|
|
7825
|
+
string role, int depth, int maxNodes, int limit, UiReadBudget budget, out int seen, out bool truncated)
|
|
7826
|
+
{
|
|
7827
|
+
int before = budget.Scanned;
|
|
7828
|
+
CacheRequest selectorCache = UiSelectorCache();
|
|
7829
|
+
var matches = UiWalkBounded(start, depth, maxNodes, limit, budget,
|
|
7830
|
+
delegate(AutomationElement element) { return ElementMatchesBudgeted(element, name, autoId, role, budget); },
|
|
7831
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element, selectorCache); },
|
|
7832
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element, selectorCache); });
|
|
7833
|
+
seen = budget.Scanned - before; truncated = budget.Truncated;
|
|
6655
7834
|
return matches;
|
|
6656
7835
|
}
|
|
6657
7836
|
|
|
@@ -6665,35 +7844,46 @@ namespace DshComputerUse.Worker
|
|
|
6665
7844
|
IntPtr root = observed == null ? FindHwnd(a) : observed.Hwnd;
|
|
6666
7845
|
if (observed != null && a.ContainsKey("hwnd") && FindHwnd(a) != root) throw new Exception("TARGET_STALE: target belongs to another window; no action sent");
|
|
6667
7846
|
if (root != IntPtr.Zero) HostGuard(root, "uiaAct");
|
|
6668
|
-
AutomationElement start = root != IntPtr.Zero ? AutomationElement.FromHandle(root) : AutomationElement.RootElement;
|
|
6669
7847
|
string name = GetS(a, "nameContains", null);
|
|
6670
7848
|
string autoId = GetS(a, "automationId", null);
|
|
6671
7849
|
string role = GetS(a, "role", null);
|
|
6672
7850
|
int index = Math.Max(0, GetI(a, "index", 0));
|
|
6673
|
-
int depth = Math.Max(1, Math.Min(24, GetI(a, "depth",
|
|
6674
|
-
int maxNodes = Math.Max(20, Math.Min(
|
|
7851
|
+
int depth = Math.Max(1, Math.Min(24, GetI(a, "depth", 6)));
|
|
7852
|
+
int maxNodes = Math.Max(20, Math.Min(1200, GetI(a, "maxNodes", 120)));
|
|
7853
|
+
bool rebind = GetB(a, "allowRebind", false);
|
|
7854
|
+
if (observed == null || rebind) ValidateUiQueryScope(root != IntPtr.Zero, true,
|
|
7855
|
+
!String.IsNullOrWhiteSpace(GetS(a, "anchor", null)), depth, maxNodes);
|
|
7856
|
+
UiReadBudget readBudget = NewUiBudget(a);
|
|
6675
7857
|
bool unique = GetB(a, "requireUnique", !a.ContainsKey("index"));
|
|
6676
7858
|
List<object> skipped = new List<object>();
|
|
6677
7859
|
int seen = 0; bool truncated = false; string resolution = null;
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
7860
|
+
List<AutomationElement> matches;
|
|
7861
|
+
try {
|
|
7862
|
+
matches = observed == null ? FindUiMatchesBudgeted(UiQueryRoot(a, root, readBudget), name, autoId, role, depth, maxNodes,
|
|
7863
|
+
unique ? 2 : index + 1, readBudget, out seen, out truncated) : new List<AutomationElement> {
|
|
7864
|
+
ResolveUiTarget(observed, rebind, depth, maxNodes, out resolution, out seen, readBudget, a) };
|
|
7865
|
+
} catch (UiBudgetStop) {
|
|
7866
|
+
throw new Exception("INCOMPLETE_UIA_SEARCH: soft provider-read budget exhausted; no action sent. Observe a narrower point/anchor or refresh once; a synchronous COM call cannot be interrupted by this budget.");
|
|
7867
|
+
}
|
|
7868
|
+
truncated = truncated || readBudget.Truncated;
|
|
7869
|
+
if (!readBudget.Continue())
|
|
7870
|
+
throw new Exception("INCOMPLETE_UIA_SEARCH: provider-read budget exhausted; no action sent; use a narrower point/anchor");
|
|
6681
7871
|
if (unique && matches.Count > 1)
|
|
6682
7872
|
throw new Exception("AMBIGUOUS_UIA_TARGET: multiple controls match; no action sent. Query computer_uia, then specify hwnd and an exact id.");
|
|
6683
7873
|
if (unique && truncated)
|
|
6684
7874
|
throw new Exception("INCOMPLETE_UIA_SEARCH: uniqueness could not be verified; no action sent. Narrow the target window or query with greater depth.");
|
|
6685
7875
|
AutomationElement found = matches.Count > index ? matches[index] : null;
|
|
6686
7876
|
if (!unique) for (int i = 0; i < Math.Min(index, matches.Count); i++)
|
|
6687
|
-
try { skipped.Add(
|
|
7877
|
+
try { skipped.Add(ElemDictBudgeted(matches[i], readBudget)); }
|
|
7878
|
+
catch (UiBudgetStop) { throw new Exception("INCOMPLETE_UIA_SEARCH: provider-read budget exhausted before action; no action sent"); } catch { }
|
|
6688
7879
|
if (found == null)
|
|
6689
7880
|
throw new Exception("uiaAct: no element matched (nameContains=" + (name ?? "-") +
|
|
6690
7881
|
", automationId=" + (autoId ?? "-") + ", role=" + (role ?? "-") +
|
|
6691
7882
|
", index=" + index + ") among " + seen + " elements");
|
|
6692
7883
|
|
|
6693
7884
|
// A UIA mutation shares the same actual-action boundary and takeover checks.
|
|
6694
|
-
//
|
|
6695
|
-
//
|
|
6696
|
-
// its automation pattern either.
|
|
7885
|
+
// Scoped selectors and observed targets still need a final process
|
|
7886
|
+
// guard; a host control is never driven through its automation pattern.
|
|
6697
7887
|
int foundPid = 0;
|
|
6698
7888
|
try { foundPid = found.Current.ProcessId; } catch { }
|
|
6699
7889
|
HostGuardPid(foundPid, "uiaAct");
|
|
@@ -7027,6 +8217,8 @@ namespace DshComputerUse.Worker
|
|
|
7027
8217
|
{
|
|
7028
8218
|
var result = Err(error.Message);
|
|
7029
8219
|
if (error.Data["glowHealth"] != null) result["glowHealth"] = error.Data["glowHealth"];
|
|
8220
|
+
foreach (string key in new string[] { "dragStarted", "expectedCursor", "actualCursor", "cursorCheck" })
|
|
8221
|
+
if (error.Data.Contains(key)) result[key] = error.Data[key];
|
|
7030
8222
|
if (error.Data["code"] != null)
|
|
7031
8223
|
{
|
|
7032
8224
|
result["code"] = (string)error.Data["code"];
|
|
@@ -7120,3 +8312,742 @@ namespace DshComputerUse.Worker
|
|
|
7120
8312
|
}
|
|
7121
8313
|
}
|
|
7122
8314
|
}
|
|
8315
|
+
|
|
8316
|
+
namespace DshComputerUse.Worker
|
|
8317
|
+
{
|
|
8318
|
+
/// A small, capture-excluded visual replacement at the native pointer hotspot.
|
|
8319
|
+
/// Native cursor resources remain intact for hit evidence and screenshot burn-in.
|
|
8320
|
+
/// Update/Hide/Dispose are called only by Glow's window-owning render thread.
|
|
8321
|
+
public static class AgentPointer
|
|
8322
|
+
{
|
|
8323
|
+
const int LogicalWidth = KitePointerPainter.LogicalWidth, LogicalHeight = KitePointerPainter.LogicalHeight;
|
|
8324
|
+
const int WS_POPUP = unchecked((int)0x80000000), WS_DISABLED = 0x08000000;
|
|
8325
|
+
const int CompanionStyle = 0x00080000 | 0x00000020 | 0x00000080 | 0x08000000 | 0x00000008;
|
|
8326
|
+
[StructLayout(LayoutKind.Sequential)] struct Pt { public int X, Y; }
|
|
8327
|
+
[StructLayout(LayoutKind.Sequential)] struct Sz { public int X, Y; }
|
|
8328
|
+
[StructLayout(LayoutKind.Sequential)] struct Rc { public int Left, Top, Right, Bottom; }
|
|
8329
|
+
[StructLayout(LayoutKind.Sequential)] struct MonitorInfo { public uint Size; public Rc Monitor, Work; public uint Flags; }
|
|
8330
|
+
[StructLayout(LayoutKind.Sequential)] struct CursorInfo { public uint Size, Flags; public IntPtr Cursor; public Pt Position; }
|
|
8331
|
+
[StructLayout(LayoutKind.Sequential, Pack = 1)] struct Blend { public byte Op, Flags, Alpha, Format; }
|
|
8332
|
+
[StructLayout(LayoutKind.Sequential)] struct BitmapHeader
|
|
8333
|
+
{
|
|
8334
|
+
public uint Size; public int Width, Height; public ushort Planes, BitCount;
|
|
8335
|
+
public uint Compression, SizeImage; public int XPelsPerMeter, YPelsPerMeter;
|
|
8336
|
+
public uint ClrUsed, ClrImportant;
|
|
8337
|
+
}
|
|
8338
|
+
[StructLayout(LayoutKind.Sequential)] struct BitmapInfo { public BitmapHeader Header; public uint Color; }
|
|
8339
|
+
[StructLayout(LayoutKind.Sequential)] struct Msg
|
|
8340
|
+
{
|
|
8341
|
+
public IntPtr Hwnd; public uint Message; public UIntPtr WParam;
|
|
8342
|
+
public IntPtr LParam; public uint Time; public Pt Point; public uint Private;
|
|
8343
|
+
}
|
|
8344
|
+
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
8345
|
+
static extern IntPtr CreateWindowEx(int ex, string cls, string title, int style, int x, int y, int w, int h, IntPtr parent, IntPtr menu, IntPtr instance, IntPtr param);
|
|
8346
|
+
[DllImport("user32.dll")] static extern bool DestroyWindow(IntPtr hwnd);
|
|
8347
|
+
[DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hwnd, int command);
|
|
8348
|
+
[DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hwnd);
|
|
8349
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool SetWindowPos(IntPtr hwnd, IntPtr after, int x, int y, int w, int h, uint flags);
|
|
8350
|
+
[DllImport("user32.dll")] static extern bool GetCursorInfo(ref CursorInfo info);
|
|
8351
|
+
[DllImport("user32.dll")] static extern IntPtr MonitorFromPoint(Pt point, uint flags);
|
|
8352
|
+
[DllImport("user32.dll")] static extern bool GetMonitorInfo(IntPtr monitor, ref MonitorInfo info);
|
|
8353
|
+
[DllImport("shcore.dll")] static extern int GetDpiForMonitor(IntPtr monitor, int kind, out uint x, out uint y);
|
|
8354
|
+
[DllImport("user32.dll")] static extern bool SetWindowDisplayAffinity(IntPtr hwnd, uint affinity);
|
|
8355
|
+
[DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr hwnd);
|
|
8356
|
+
[DllImport("user32.dll")] static extern int ReleaseDC(IntPtr hwnd, IntPtr dc);
|
|
8357
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern bool PeekMessage(out Msg msg, IntPtr hwnd, uint min, uint max, uint remove);
|
|
8358
|
+
[DllImport("user32.dll")] static extern bool ValidateRect(IntPtr hwnd, IntPtr rect);
|
|
8359
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern IntPtr DispatchMessage(ref Msg msg);
|
|
8360
|
+
[DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr dc);
|
|
8361
|
+
[DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr dc);
|
|
8362
|
+
[DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr dc, IntPtr value);
|
|
8363
|
+
[DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr value);
|
|
8364
|
+
[DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr dc, ref BitmapInfo info, uint use, out IntPtr bits, IntPtr section, uint offset);
|
|
8365
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool UpdateLayeredWindow(IntPtr hwnd, IntPtr dc, ref Pt dest, ref Sz size, IntPtr source, ref Pt origin, int key, ref Blend blend, int flags);
|
|
8366
|
+
|
|
8367
|
+
static IntPtr window, screenDc, memoryDc, dib, oldBitmap, pixels, monitor;
|
|
8368
|
+
static int width, height, lastX = int.MinValue, lastY = int.MinValue, lastFrame = -1;
|
|
8369
|
+
static bool visible, failed, lastHolding;
|
|
8370
|
+
static string lastShape;
|
|
8371
|
+
static float scale = 1;
|
|
8372
|
+
static Rc monitorBounds;
|
|
8373
|
+
static byte[] copy;
|
|
8374
|
+
public static string Fault { get; private set; }
|
|
8375
|
+
|
|
8376
|
+
// Only a protected hold animates. Sixteen brightness steps over 2.5 seconds
|
|
8377
|
+
// avoid a repaint on every Glow tick; active input has a static bitmap.
|
|
8378
|
+
public static int AnimationFrame(bool holding, int tick)
|
|
8379
|
+
{
|
|
8380
|
+
return KitePointerPainter.AnimationFrame(holding, tick);
|
|
8381
|
+
}
|
|
8382
|
+
|
|
8383
|
+
public static void Update(bool show, bool holding)
|
|
8384
|
+
{
|
|
8385
|
+
if (!show || failed) { Hide(); return; }
|
|
8386
|
+
try
|
|
8387
|
+
{
|
|
8388
|
+
CursorInfo cursor = new CursorInfo(); cursor.Size = (uint)Marshal.SizeOf(typeof(CursorInfo));
|
|
8389
|
+
// Follow native cursor visibility, including text-entry apps which hide it.
|
|
8390
|
+
if (!GetCursorInfo(ref cursor) || (cursor.Flags & 1) == 0) { Hide(); return; }
|
|
8391
|
+
IntPtr nextMonitor = MonitorFromPoint(cursor.Position, 2);
|
|
8392
|
+
if (nextMonitor != monitor || window == IntPtr.Zero)
|
|
8393
|
+
{
|
|
8394
|
+
MonitorInfo info = new MonitorInfo(); info.Size = (uint)Marshal.SizeOf(typeof(MonitorInfo));
|
|
8395
|
+
if (!GetMonitorInfo(nextMonitor, ref info)) { Hide(); return; }
|
|
8396
|
+
monitorBounds = info.Monitor; monitor = nextMonitor;
|
|
8397
|
+
uint dpiX = 96, dpiY = 96;
|
|
8398
|
+
try { if (GetDpiForMonitor(monitor, 0, out dpiX, out dpiY) != 0) dpiX = 96; }
|
|
8399
|
+
catch (EntryPointNotFoundException) { dpiX = 96; }
|
|
8400
|
+
catch (DllNotFoundException) { dpiX = 96; }
|
|
8401
|
+
scale = Math.Max(1.0f, Math.Min(4.0f, dpiX / 96.0f));
|
|
8402
|
+
EnsureSurface((int)Math.Ceiling(LogicalWidth * scale), (int)Math.Ceiling(LogicalHeight * scale));
|
|
8403
|
+
lastFrame = -1;
|
|
8404
|
+
}
|
|
8405
|
+
// The drawn tip IS the native click point. Never flip or clamp the
|
|
8406
|
+
// surface near an edge: that would lie about the actual hotspot.
|
|
8407
|
+
System.Drawing.Point hotspot = KitePointerPainter.Hotspot(scale);
|
|
8408
|
+
int x = cursor.Position.X - hotspot.X, y = cursor.Position.Y - hotspot.Y;
|
|
8409
|
+
string shape = Program.StandardCursorShape(cursor.Cursor,
|
|
8410
|
+
delegate(int id) { return Native.LoadCursor(IntPtr.Zero, new IntPtr(id)); });
|
|
8411
|
+
int frame = AnimationFrame(holding, Environment.TickCount);
|
|
8412
|
+
if (frame != lastFrame || holding != lastHolding || shape != lastShape)
|
|
8413
|
+
{
|
|
8414
|
+
using (Bitmap bitmap = RenderPreview(scale, holding, frame, shape))
|
|
8415
|
+
{
|
|
8416
|
+
BitmapData data = bitmap.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, PixelFormat.Format32bppPArgb);
|
|
8417
|
+
try
|
|
8418
|
+
{
|
|
8419
|
+
if (copy == null || copy.Length != width * height * 4) copy = new byte[width * height * 4];
|
|
8420
|
+
for (int row = 0; row < height; row++)
|
|
8421
|
+
Marshal.Copy(IntPtr.Add(data.Scan0, row * data.Stride), copy, row * width * 4, width * 4);
|
|
8422
|
+
Marshal.Copy(copy, 0, pixels, copy.Length);
|
|
8423
|
+
}
|
|
8424
|
+
finally { bitmap.UnlockBits(data); }
|
|
8425
|
+
}
|
|
8426
|
+
Pt target = new Pt { X = x, Y = y }, origin = new Pt();
|
|
8427
|
+
Sz size = new Sz { X = width, Y = height };
|
|
8428
|
+
Blend blend = new Blend { Alpha = 255, Format = 1 };
|
|
8429
|
+
if (!UpdateLayeredWindow(window, screenDc, ref target, ref size, memoryDc, ref origin, 0, ref blend, 2))
|
|
8430
|
+
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8431
|
+
lastFrame = frame; lastHolding = holding; lastShape = shape; lastX = x; lastY = y;
|
|
8432
|
+
}
|
|
8433
|
+
else if (x != lastX || y != lastY)
|
|
8434
|
+
{
|
|
8435
|
+
// Moving a cached tiny layered surface does not redraw the desktop.
|
|
8436
|
+
if (!SetWindowPos(window, new IntPtr(-1), x, y, 0, 0, 0x0001 | 0x0010))
|
|
8437
|
+
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8438
|
+
lastX = x; lastY = y;
|
|
8439
|
+
}
|
|
8440
|
+
if (!visible) { ShowWindow(window, 4); visible = true; }
|
|
8441
|
+
if (!IsWindowVisible(window)) throw new InvalidOperationException("replacement pointer surface is not visible");
|
|
8442
|
+
// Publish a short hide lease only AFTER a successful visible paint.
|
|
8443
|
+
// While the independent guardian starts, retain the native cursor.
|
|
8444
|
+
if (!RequestReplacement(true))
|
|
8445
|
+
{
|
|
8446
|
+
ShowWindow(window, 0); visible = false;
|
|
8447
|
+
if (PointerVisibilityLease.Fault != null) throw new InvalidOperationException(PointerVisibilityLease.Fault);
|
|
8448
|
+
return;
|
|
8449
|
+
}
|
|
8450
|
+
Msg msg;
|
|
8451
|
+
for (int n = 0; n < 8 && PeekMessage(out msg, window, 0, 0, 1); n++)
|
|
8452
|
+
{
|
|
8453
|
+
if (msg.Message == 0x000F) ValidateRect(window, IntPtr.Zero);
|
|
8454
|
+
else DispatchMessage(ref msg);
|
|
8455
|
+
}
|
|
8456
|
+
}
|
|
8457
|
+
catch (Exception error)
|
|
8458
|
+
{
|
|
8459
|
+
// This presentation is optional. A display failure must never block input,
|
|
8460
|
+
// leak a global cursor, or turn a recoverable Glow frame into a fatal one.
|
|
8461
|
+
Fault = error.GetType().Name + ": " + error.Message; failed = true; Dispose();
|
|
8462
|
+
}
|
|
8463
|
+
}
|
|
8464
|
+
|
|
8465
|
+
static void EnsureSurface(int w, int h)
|
|
8466
|
+
{
|
|
8467
|
+
if (window == IntPtr.Zero)
|
|
8468
|
+
{
|
|
8469
|
+
window = CreateWindowEx(CompanionStyle, "STATIC", "DSH kite pointer", WS_POPUP | WS_DISABLED,
|
|
8470
|
+
0, 0, w, h, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
|
|
8471
|
+
if (window == IntPtr.Zero) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8472
|
+
// If exclusion is unavailable, omit the decoration rather than paint it
|
|
8473
|
+
// into evidence. WS_DISABLED + TRANSPARENT prevent input interception.
|
|
8474
|
+
if (!SetWindowDisplayAffinity(window, 0x11)) throw new InvalidOperationException("pointer capture exclusion unavailable");
|
|
8475
|
+
screenDc = GetDC(IntPtr.Zero); memoryDc = CreateCompatibleDC(screenDc);
|
|
8476
|
+
if (screenDc == IntPtr.Zero || memoryDc == IntPtr.Zero) throw new InvalidOperationException("pointer DC unavailable");
|
|
8477
|
+
}
|
|
8478
|
+
if (width == w && height == h && dib != IntPtr.Zero) return;
|
|
8479
|
+
ReleaseSurface();
|
|
8480
|
+
BitmapInfo info = new BitmapInfo(); info.Header.Size = (uint)Marshal.SizeOf(typeof(BitmapHeader));
|
|
8481
|
+
info.Header.Width = w; info.Header.Height = -h; info.Header.Planes = 1; info.Header.BitCount = 32;
|
|
8482
|
+
dib = CreateDIBSection(screenDc, ref info, 0, out pixels, IntPtr.Zero, 0);
|
|
8483
|
+
if (dib == IntPtr.Zero) throw new InvalidOperationException("pointer surface unavailable");
|
|
8484
|
+
oldBitmap = SelectObject(memoryDc, dib);
|
|
8485
|
+
if (oldBitmap == IntPtr.Zero || oldBitmap == new IntPtr(-1)) throw new InvalidOperationException("pointer bitmap unavailable");
|
|
8486
|
+
width = w; height = h;
|
|
8487
|
+
}
|
|
8488
|
+
|
|
8489
|
+
// Offscreen previews and the live pointer share the exact painter/hotspot.
|
|
8490
|
+
public static Bitmap RenderPreview(float dpiScale, bool holding, int frame, string semantic = "arrow")
|
|
8491
|
+
{ return KitePointerPainter.RenderPreview(dpiScale, holding, frame, semantic); }
|
|
8492
|
+
|
|
8493
|
+
// A narrow seam for private-desktop tests; production uses the guardian lease.
|
|
8494
|
+
static bool RequestReplacement(bool show)
|
|
8495
|
+
{ if (show) return PointerVisibilityLease.TryHide(); PointerVisibilityLease.Show(); return false; }
|
|
8496
|
+
|
|
8497
|
+
public static void Hide()
|
|
8498
|
+
{
|
|
8499
|
+
RequestReplacement(false);
|
|
8500
|
+
if (window != IntPtr.Zero && visible) ShowWindow(window, 0);
|
|
8501
|
+
visible = false;
|
|
8502
|
+
}
|
|
8503
|
+
static void ReleaseSurface()
|
|
8504
|
+
{
|
|
8505
|
+
if (oldBitmap != IntPtr.Zero && oldBitmap != new IntPtr(-1) && memoryDc != IntPtr.Zero) SelectObject(memoryDc, oldBitmap);
|
|
8506
|
+
oldBitmap = IntPtr.Zero;
|
|
8507
|
+
if (dib != IntPtr.Zero) DeleteObject(dib);
|
|
8508
|
+
dib = IntPtr.Zero; pixels = IntPtr.Zero; width = height = 0; copy = null;
|
|
8509
|
+
}
|
|
8510
|
+
public static void Dispose()
|
|
8511
|
+
{
|
|
8512
|
+
Hide(); ReleaseSurface();
|
|
8513
|
+
PointerVisibilityLease.Dispose();
|
|
8514
|
+
if (memoryDc != IntPtr.Zero) DeleteDC(memoryDc);
|
|
8515
|
+
if (screenDc != IntPtr.Zero) ReleaseDC(IntPtr.Zero, screenDc);
|
|
8516
|
+
if (window != IntPtr.Zero) DestroyWindow(window);
|
|
8517
|
+
memoryDc = screenDc = window = monitor = IntPtr.Zero;
|
|
8518
|
+
lastFrame = -1; lastX = lastY = int.MinValue;
|
|
8519
|
+
}
|
|
8520
|
+
}
|
|
8521
|
+
}
|
|
8522
|
+
|
|
8523
|
+
namespace DshComputerUse.Worker
|
|
8524
|
+
{
|
|
8525
|
+
public static class KitePointerPainter
|
|
8526
|
+
{
|
|
8527
|
+
public const int LogicalWidth = 39, LogicalHeight = 44;
|
|
8528
|
+
public const float BodyScale = 0.75f, TiltDegrees = -25f;
|
|
8529
|
+
public static readonly PointF LogicalHotspot = new PointF(8, 4);
|
|
8530
|
+
static readonly double Angle = TiltDegrees * Math.PI / 180.0;
|
|
8531
|
+
|
|
8532
|
+
static float Scale(float value)
|
|
8533
|
+
{
|
|
8534
|
+
if (float.IsNaN(value) || float.IsInfinity(value)) return 1f;
|
|
8535
|
+
return Math.Max(1f, Math.Min(4f, value));
|
|
8536
|
+
}
|
|
8537
|
+
|
|
8538
|
+
public static Size CanvasSize(float dpiScale)
|
|
8539
|
+
{
|
|
8540
|
+
float s = Scale(dpiScale);
|
|
8541
|
+
return new Size((int)Math.Ceiling(LogicalWidth * s), (int)Math.Ceiling(LogicalHeight * s));
|
|
8542
|
+
}
|
|
8543
|
+
|
|
8544
|
+
public static Point Hotspot(float dpiScale)
|
|
8545
|
+
{
|
|
8546
|
+
float s = Scale(dpiScale);
|
|
8547
|
+
return new Point((int)Math.Round(LogicalHotspot.X * s, MidpointRounding.AwayFromZero),
|
|
8548
|
+
(int)Math.Round(LogicalHotspot.Y * s, MidpointRounding.AwayFromZero));
|
|
8549
|
+
}
|
|
8550
|
+
|
|
8551
|
+
// Source kite tip is (27,7). Rotation is counterclockwise in screen coordinates:
|
|
8552
|
+
// an upward-pointing head tilts toward the upper-left. Device-pixel H is shared
|
|
8553
|
+
// by the painter and the future cursor hotspot; it never depends on animation.
|
|
8554
|
+
public static PointF Transform(float x, float y, float dpiScale)
|
|
8555
|
+
{
|
|
8556
|
+
float s = Scale(dpiScale);
|
|
8557
|
+
Point h = Hotspot(s);
|
|
8558
|
+
double dx = x - 27, dy = y - 7;
|
|
8559
|
+
return new PointF(h.X + (float)(s * BodyScale * (Math.Cos(Angle) * dx - Math.Sin(Angle) * dy)),
|
|
8560
|
+
h.Y + (float)(s * BodyScale * (Math.Sin(Angle) * dx + Math.Cos(Angle) * dy)));
|
|
8561
|
+
}
|
|
8562
|
+
|
|
8563
|
+
public static int AnimationFrame(bool holding, int tick)
|
|
8564
|
+
{
|
|
8565
|
+
if (!holding) return 16;
|
|
8566
|
+
double phase = (unchecked((uint)tick) % 2500) * 2 * Math.PI / 2500.0;
|
|
8567
|
+
return 7 + (int)Math.Round(4 * (1 - Math.Cos(phase)));
|
|
8568
|
+
}
|
|
8569
|
+
|
|
8570
|
+
static PointF[] Shape(float s, params float[] xy)
|
|
8571
|
+
{
|
|
8572
|
+
PointF[] points = new PointF[xy.Length / 2];
|
|
8573
|
+
for (int i = 0; i < points.Length; i++) points[i] = Transform(xy[2 * i], xy[2 * i + 1], s);
|
|
8574
|
+
return points;
|
|
8575
|
+
}
|
|
8576
|
+
|
|
8577
|
+
public static Bitmap RenderPreview(float dpiScale, bool holding, int frame, string semantic = "arrow")
|
|
8578
|
+
{
|
|
8579
|
+
float s = Scale(dpiScale);
|
|
8580
|
+
Size size = CanvasSize(s);
|
|
8581
|
+
Bitmap bitmap = new Bitmap(size.Width, size.Height, PixelFormat.Format32bppPArgb);
|
|
8582
|
+
float pulse = holding ? Math.Max(0f, Math.Min(1f, (frame - 7) / 8f)) : 1f;
|
|
8583
|
+
int bodyAlpha = holding ? 208 + (int)(36 * pulse) : 248;
|
|
8584
|
+
PointF[] kite = Shape(s, 27, 7, 41, 24, 27, 43, 13, 24);
|
|
8585
|
+
using (Graphics g = Graphics.FromImage(bitmap))
|
|
8586
|
+
{
|
|
8587
|
+
g.Clear(Color.Transparent);
|
|
8588
|
+
g.SmoothingMode = SmoothingMode.AntiAlias;
|
|
8589
|
+
g.PixelOffsetMode = PixelOffsetMode.HighQuality;
|
|
8590
|
+
|
|
8591
|
+
// A slim visible string attaches to the lower point. No satellite star,
|
|
8592
|
+
// duplicate arrow, end bead or external status symbol crowds the tip.
|
|
8593
|
+
using (Pen tail = new Pen(Color.FromArgb(holding ? 185 + (int)(35 * pulse) : 230, 39, 118, 224), 1.1f * s))
|
|
8594
|
+
{
|
|
8595
|
+
tail.StartCap = tail.EndCap = LineCap.Round;
|
|
8596
|
+
g.DrawBezier(tail, Transform(27, 43, s), Transform(25, 53, s),
|
|
8597
|
+
Transform(11, 41, s), Transform(10, 53, s));
|
|
8598
|
+
}
|
|
8599
|
+
|
|
8600
|
+
using (Pen halo = new Pen(Color.FromArgb(holding ? 8 + (int)(20 * pulse) : 28, 54, 132, 255), 2.5f * s))
|
|
8601
|
+
g.DrawPolygon(halo, kite);
|
|
8602
|
+
using (LinearGradientBrush fill = new LinearGradientBrush(Transform(22, 9, s), Transform(33, 43, s),
|
|
8603
|
+
Color.FromArgb(bodyAlpha, holding ? 49 : 57, holding ? 133 : 150, 250),
|
|
8604
|
+
Color.FromArgb(bodyAlpha, 20, 51, holding ? 128 : 151)))
|
|
8605
|
+
g.FillPolygon(fill, kite);
|
|
8606
|
+
using (SolidBrush facet = new SolidBrush(Color.FromArgb(holding ? 78 + (int)(25 * pulse) : 115, 160, 225, 255)))
|
|
8607
|
+
g.FillPolygon(facet, Shape(s, 27, 7, 41, 24, 27, 25));
|
|
8608
|
+
using (SolidBrush shade = new SolidBrush(Color.FromArgb(78, 6, 24, 79)))
|
|
8609
|
+
g.FillPolygon(shade, Shape(s, 27, 25, 41, 24, 27, 43));
|
|
8610
|
+
using (Pen rim = new Pen(Color.FromArgb(242, 191, 231, 255), Math.Max(0.85f, 0.85f * s)))
|
|
8611
|
+
{
|
|
8612
|
+
rim.LineJoin = LineJoin.Miter;
|
|
8613
|
+
g.DrawPolygon(rim, kite);
|
|
8614
|
+
}
|
|
8615
|
+
using (Pen spine = new Pen(Color.FromArgb(72, 213, 242, 255), 0.55f * s))
|
|
8616
|
+
g.DrawLine(spine, Transform(27, 8, s), Transform(27, 42, s));
|
|
8617
|
+
|
|
8618
|
+
// The state mark stays legible at the dimmest hold frame. Geometry and
|
|
8619
|
+
// hotspot remain stationary; only light/alpha changes during breathing.
|
|
8620
|
+
if (NormalizeSemantic(semantic) == "arrow")
|
|
8621
|
+
using (SolidBrush mark = new SolidBrush(Color.FromArgb(245, 236, 249, 255)))
|
|
8622
|
+
{
|
|
8623
|
+
if (holding)
|
|
8624
|
+
{
|
|
8625
|
+
g.FillPolygon(mark, Shape(s, 23.3f, 20.2f, 25.2f, 20.2f, 25.2f, 28.2f, 23.3f, 28.2f));
|
|
8626
|
+
g.FillPolygon(mark, Shape(s, 29.1f, 20.2f, 31.0f, 20.2f, 31.0f, 28.2f, 29.1f, 28.2f));
|
|
8627
|
+
}
|
|
8628
|
+
else g.FillPolygon(mark, Shape(s, 24, 19.8f, 31.4f, 24.2f, 24, 28.6f));
|
|
8629
|
+
}
|
|
8630
|
+
DrawSemantic(g, s, semantic);
|
|
8631
|
+
}
|
|
8632
|
+
return bitmap;
|
|
8633
|
+
}
|
|
8634
|
+
|
|
8635
|
+
public static string NormalizeSemantic(string semantic)
|
|
8636
|
+
{
|
|
8637
|
+
if (String.IsNullOrEmpty(semantic)) return "arrow";
|
|
8638
|
+
string value = semantic.Trim().ToLowerInvariant();
|
|
8639
|
+
if (value == "ibeam" || value == "i-beam") return "text";
|
|
8640
|
+
switch (value)
|
|
8641
|
+
{
|
|
8642
|
+
case "arrow": case "text": case "resize-ew": case "resize-ns":
|
|
8643
|
+
case "resize-nwse": case "resize-nesw": case "move": case "hand":
|
|
8644
|
+
case "cross": case "wait": case "app-starting": case "not-allowed": case "help":
|
|
8645
|
+
return value;
|
|
8646
|
+
default: return "unknown";
|
|
8647
|
+
}
|
|
8648
|
+
}
|
|
8649
|
+
|
|
8650
|
+
// Every semantic is a small upright center mark. Neither the canvas, kite,
|
|
8651
|
+
// tail nor hotspot moves; holding keeps the same mark and only breathes.
|
|
8652
|
+
// An unknown native cursor gets a neutral dash, never a guessed resize/text mark.
|
|
8653
|
+
static void DrawSemantic(Graphics g, float s, string semantic)
|
|
8654
|
+
{
|
|
8655
|
+
semantic = NormalizeSemantic(semantic);
|
|
8656
|
+
if (semantic == "arrow") return;
|
|
8657
|
+
PointF center = Transform(27, 25, s);
|
|
8658
|
+
using (GraphicsPath path = new GraphicsPath())
|
|
8659
|
+
using (Matrix matrix = new Matrix(s, 0, 0, s, center.X, center.Y))
|
|
8660
|
+
{
|
|
8661
|
+
if (semantic == "text")
|
|
8662
|
+
{
|
|
8663
|
+
path.AddLine(-2, -4, 2, -4);
|
|
8664
|
+
path.StartFigure(); path.AddLine(0, -4, 0, 4);
|
|
8665
|
+
path.StartFigure(); path.AddLine(-2, 4, 2, 4);
|
|
8666
|
+
}
|
|
8667
|
+
else if (semantic.StartsWith("resize-"))
|
|
8668
|
+
{
|
|
8669
|
+
float angle = semantic == "resize-ns" ? 90 : semantic == "resize-nwse" ? 45 : semantic == "resize-nesw" ? -45 : 0;
|
|
8670
|
+
AddDoubleArrow(path, 4.0f, 1.9f);
|
|
8671
|
+
using (Matrix rotate = new Matrix()) { rotate.Rotate(angle); path.Transform(rotate); }
|
|
8672
|
+
}
|
|
8673
|
+
else if (semantic == "move")
|
|
8674
|
+
{
|
|
8675
|
+
AddDoubleArrow(path, 3.9f, 1.5f);
|
|
8676
|
+
using (GraphicsPath vertical = new GraphicsPath())
|
|
8677
|
+
using (Matrix rotate = new Matrix())
|
|
8678
|
+
{
|
|
8679
|
+
AddDoubleArrow(vertical, 3.9f, 1.5f); rotate.Rotate(90); vertical.Transform(rotate);
|
|
8680
|
+
path.AddPath(vertical, false);
|
|
8681
|
+
}
|
|
8682
|
+
}
|
|
8683
|
+
else if (semantic == "cross")
|
|
8684
|
+
{
|
|
8685
|
+
path.AddLine(-4, 0, 4, 0); path.StartFigure(); path.AddLine(0, -4, 0, 4);
|
|
8686
|
+
}
|
|
8687
|
+
else if (semantic == "not-allowed")
|
|
8688
|
+
{
|
|
8689
|
+
path.AddEllipse(-3.6f, -3.6f, 7.2f, 7.2f);
|
|
8690
|
+
path.StartFigure(); path.AddLine(-2.5f, 2.5f, 2.5f, -2.5f);
|
|
8691
|
+
}
|
|
8692
|
+
else if (semantic == "wait")
|
|
8693
|
+
{
|
|
8694
|
+
AddHourglass(path, 0, 0, 2.6f, 3.8f);
|
|
8695
|
+
}
|
|
8696
|
+
else if (semantic == "app-starting")
|
|
8697
|
+
{
|
|
8698
|
+
path.AddLines(new PointF[] { new PointF(-3.5f, 2.7f), new PointF(-3.5f, -3.8f), new PointF(-0.2f, -0.4f) });
|
|
8699
|
+
path.StartFigure(); AddHourglass(path, 1.8f, 1.2f, 1.5f, 2.6f);
|
|
8700
|
+
}
|
|
8701
|
+
else if (semantic == "help")
|
|
8702
|
+
{
|
|
8703
|
+
path.AddBezier(-2.4f, -2.4f, -2.0f, -5.3f, 4.1f, -4.0f, 1.7f, -1.3f);
|
|
8704
|
+
path.AddBezier(1.7f, -1.3f, 0.6f, -0.5f, 0, -0.4f, 0, 1.2f);
|
|
8705
|
+
path.StartFigure(); path.AddLine(0, 3.3f, 0, 3.6f);
|
|
8706
|
+
}
|
|
8707
|
+
else if (semantic == "hand")
|
|
8708
|
+
{
|
|
8709
|
+
path.AddPolygon(new PointF[] { new PointF(-3.7f, 0), new PointF(-2.4f, -1),
|
|
8710
|
+
new PointF(-1, 0.4f), new PointF(-1, -4.2f), new PointF(0.6f, -4.2f),
|
|
8711
|
+
new PointF(0.6f, -0.8f), new PointF(1.8f, -1.4f), new PointF(3, -0.5f),
|
|
8712
|
+
new PointF(3.7f, 0.6f), new PointF(3.1f, 3.6f), new PointF(-0.2f, 3.9f), new PointF(-2, 2) });
|
|
8713
|
+
}
|
|
8714
|
+
else
|
|
8715
|
+
{
|
|
8716
|
+
path.AddLine(-2.1f, 0, 2.1f, 0);
|
|
8717
|
+
}
|
|
8718
|
+
path.Transform(matrix);
|
|
8719
|
+
using (Pen contrast = new Pen(Color.FromArgb(110, 8, 32, 89), 2.0f * s))
|
|
8720
|
+
using (Pen ink = new Pen(Color.FromArgb(semantic == "unknown" ? 180 : 248, 232, 247, 255), 1.0f * s))
|
|
8721
|
+
{
|
|
8722
|
+
contrast.LineJoin = ink.LineJoin = LineJoin.Round;
|
|
8723
|
+
contrast.StartCap = contrast.EndCap = ink.StartCap = ink.EndCap = LineCap.Round;
|
|
8724
|
+
g.DrawPath(contrast, path); g.DrawPath(ink, path);
|
|
8725
|
+
}
|
|
8726
|
+
}
|
|
8727
|
+
}
|
|
8728
|
+
|
|
8729
|
+
static void AddDoubleArrow(GraphicsPath path, float radius, float head)
|
|
8730
|
+
{
|
|
8731
|
+
path.AddLine(-radius, 0, radius, 0); path.StartFigure();
|
|
8732
|
+
path.AddLines(new PointF[] { new PointF(-radius + head, -head), new PointF(-radius, 0), new PointF(-radius + head, head) });
|
|
8733
|
+
path.StartFigure();
|
|
8734
|
+
path.AddLines(new PointF[] { new PointF(radius - head, -head), new PointF(radius, 0), new PointF(radius - head, head) });
|
|
8735
|
+
}
|
|
8736
|
+
|
|
8737
|
+
static void AddHourglass(GraphicsPath path, float x, float y, float width, float height)
|
|
8738
|
+
{
|
|
8739
|
+
path.AddLines(new PointF[] { new PointF(x - width, y - height), new PointF(x + width, y - height),
|
|
8740
|
+
new PointF(x - width, y + height), new PointF(x + width, y + height), new PointF(x - width, y - height) });
|
|
8741
|
+
}
|
|
8742
|
+
}
|
|
8743
|
+
|
|
8744
|
+
}
|
|
8745
|
+
|
|
8746
|
+
namespace DshComputerUse.Worker
|
|
8747
|
+
{
|
|
8748
|
+
public static class PointerVisibilityLease
|
|
8749
|
+
{
|
|
8750
|
+
const int Version = 1, Bytes = 128, ReadyOffset = 48, RestoredOffset = 52;
|
|
8751
|
+
const int LeaseMs = 500, RefreshMs = 100, StartupMs = 2000;
|
|
8752
|
+
const string Prefix = "Local\\DSH-CursorLease-";
|
|
8753
|
+
static readonly object Gate = new object();
|
|
8754
|
+
static Session current;
|
|
8755
|
+
static string fault;
|
|
8756
|
+
public static string Fault { get { lock (Gate) return fault; } }
|
|
8757
|
+
internal static long NowMs()
|
|
8758
|
+
{ return (long)(System.Diagnostics.Stopwatch.GetTimestamp() * (1000.0 / System.Diagnostics.Stopwatch.Frequency)); }
|
|
8759
|
+
|
|
8760
|
+
sealed class Session
|
|
8761
|
+
{
|
|
8762
|
+
public System.IO.MemoryMappedFiles.MemoryMappedFile Map;
|
|
8763
|
+
public System.IO.MemoryMappedFiles.MemoryMappedViewAccessor View;
|
|
8764
|
+
public System.Diagnostics.Process Guardian;
|
|
8765
|
+
public bool Ready, Revoked, Requested, Finished;
|
|
8766
|
+
public long Sequence, LastRefresh;
|
|
8767
|
+
}
|
|
8768
|
+
|
|
8769
|
+
// Invoke ONLY after the overlay has been successfully presented at the
|
|
8770
|
+
// native hotspot. True means the ready guardian accepted a current lease,
|
|
8771
|
+
// not proof that pixels disappeared. Never blocks waiting for readiness.
|
|
8772
|
+
public static bool TryHide()
|
|
8773
|
+
{
|
|
8774
|
+
lock (Gate)
|
|
8775
|
+
{
|
|
8776
|
+
if (fault != null) return false;
|
|
8777
|
+
if (current != null && current.Finished) current = null;
|
|
8778
|
+
if (current == null)
|
|
8779
|
+
{
|
|
8780
|
+
current = new Session(); Session launch = current;
|
|
8781
|
+
System.Threading.ThreadPool.QueueUserWorkItem(delegate { StartAndMonitor(launch); });
|
|
8782
|
+
return false;
|
|
8783
|
+
}
|
|
8784
|
+
Session s = current;
|
|
8785
|
+
if (s.Revoked || !s.Ready || s.View == null) return false;
|
|
8786
|
+
long now = NowMs();
|
|
8787
|
+
if (!s.Requested || now - s.LastRefresh >= RefreshMs)
|
|
8788
|
+
{
|
|
8789
|
+
s.Requested = true; // Set before writing: unknown write outcome also needs cleanup.
|
|
8790
|
+
try { WriteRequest(s, true, false, now); s.LastRefresh = now; }
|
|
8791
|
+
catch (System.Exception e) { fault = "cursor lease write: " + e.Message; s.Revoked = true; return false; }
|
|
8792
|
+
}
|
|
8793
|
+
return true;
|
|
8794
|
+
}
|
|
8795
|
+
}
|
|
8796
|
+
|
|
8797
|
+
// A nonblocking release request. Guardian observes it within its next
|
|
8798
|
+
// progressing <=20 ms poll; it is not a synchronous compositor fence.
|
|
8799
|
+
public static void Show()
|
|
8800
|
+
{
|
|
8801
|
+
lock (Gate)
|
|
8802
|
+
{
|
|
8803
|
+
Session s = current;
|
|
8804
|
+
if (s == null || s.Finished || s.Revoked) return;
|
|
8805
|
+
s.Revoked = true;
|
|
8806
|
+
if (s.View != null)
|
|
8807
|
+
{
|
|
8808
|
+
try { WriteRequest(s, false, true, NowMs()); }
|
|
8809
|
+
catch (System.Exception e) { fault = "cursor lease release: " + e.Message; }
|
|
8810
|
+
}
|
|
8811
|
+
}
|
|
8812
|
+
}
|
|
8813
|
+
public static void Dispose() { Show(); }
|
|
8814
|
+
|
|
8815
|
+
static void WriteRequest(Session s, bool desired, bool revoked, long stamp)
|
|
8816
|
+
{
|
|
8817
|
+
long even = s.Sequence + 2;
|
|
8818
|
+
s.View.Write(16, even - 1); System.Threading.Thread.MemoryBarrier();
|
|
8819
|
+
s.View.Write(24, desired ? 1 : 0); s.View.Write(28, revoked ? 1 : 0); s.View.Write(32, stamp);
|
|
8820
|
+
System.Threading.Thread.MemoryBarrier(); s.View.Write(16, even); s.Sequence = even;
|
|
8821
|
+
}
|
|
8822
|
+
internal sealed class Request
|
|
8823
|
+
{
|
|
8824
|
+
public long Sequence, Stamp;
|
|
8825
|
+
public bool Desired, Revoked;
|
|
8826
|
+
public Request(long sequence, long stamp, bool desired, bool revoked)
|
|
8827
|
+
{ Sequence = sequence; Stamp = stamp; Desired = desired; Revoked = revoked; }
|
|
8828
|
+
}
|
|
8829
|
+
internal static Request ReadRequest(System.IO.MemoryMappedFiles.MemoryMappedViewAccessor view)
|
|
8830
|
+
{
|
|
8831
|
+
long first = view.ReadInt64(16);
|
|
8832
|
+
if (first <= 0 || (first & 1) != 0) return null;
|
|
8833
|
+
System.Threading.Thread.MemoryBarrier();
|
|
8834
|
+
int desired = view.ReadInt32(24), revoked = view.ReadInt32(28); long stamp = view.ReadInt64(32);
|
|
8835
|
+
System.Threading.Thread.MemoryBarrier(); long second = view.ReadInt64(16);
|
|
8836
|
+
if (first != second || (desired != 0 && desired != 1) || (revoked != 0 && revoked != 1)) return null;
|
|
8837
|
+
return new Request(second, stamp, desired == 1, revoked == 1);
|
|
8838
|
+
}
|
|
8839
|
+
|
|
8840
|
+
static void StartAndMonitor(Session s)
|
|
8841
|
+
{
|
|
8842
|
+
System.IO.MemoryMappedFiles.MemoryMappedFile map = null;
|
|
8843
|
+
System.IO.MemoryMappedFiles.MemoryMappedViewAccessor view = null;
|
|
8844
|
+
System.Diagnostics.Process child = null;
|
|
8845
|
+
bool restoreNeeded = false, exitConfirmed = false, stopAttempted = false;
|
|
8846
|
+
try
|
|
8847
|
+
{
|
|
8848
|
+
string name = Prefix + System.Guid.NewGuid().ToString("N");
|
|
8849
|
+
using (System.Diagnostics.Process owner = System.Diagnostics.Process.GetCurrentProcess())
|
|
8850
|
+
{
|
|
8851
|
+
map = System.IO.MemoryMappedFiles.MemoryMappedFile.CreateNew(name, Bytes);
|
|
8852
|
+
view = map.CreateViewAccessor(0, Bytes);
|
|
8853
|
+
view.Write(0, Version); view.Write(4, owner.Id);
|
|
8854
|
+
long start = owner.StartTime.ToUniversalTime().Ticks; view.Write(8, start);
|
|
8855
|
+
lock (Gate)
|
|
8856
|
+
{
|
|
8857
|
+
s.Map = map; s.View = view;
|
|
8858
|
+
if (s.Revoked) return; // No child and no Mag API if released during startup.
|
|
8859
|
+
WriteRequest(s, false, false, NowMs());
|
|
8860
|
+
}
|
|
8861
|
+
var info = new System.Diagnostics.ProcessStartInfo(
|
|
8862
|
+
System.Reflection.Assembly.GetExecutingAssembly().Location,
|
|
8863
|
+
"--pointer-guardian " + name + " " + owner.Id + " " + start);
|
|
8864
|
+
info.UseShellExecute = false; info.CreateNoWindow = true;
|
|
8865
|
+
info.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
|
|
8866
|
+
child = System.Diagnostics.Process.Start(info);
|
|
8867
|
+
if (child == null) throw new System.InvalidOperationException("guardian did not start");
|
|
8868
|
+
lock (Gate) s.Guardian = child;
|
|
8869
|
+
}
|
|
8870
|
+
long began = NowMs();
|
|
8871
|
+
while (!child.WaitForExit(20))
|
|
8872
|
+
{
|
|
8873
|
+
int status = view.ReadInt32(ReadyOffset);
|
|
8874
|
+
lock (Gate) s.Ready = status == 1 && !s.Revoked;
|
|
8875
|
+
if (status == 0 && NowMs() - began > StartupMs)
|
|
8876
|
+
{
|
|
8877
|
+
// Only our own Process handle is terminated. It was not
|
|
8878
|
+
// ready, so TryHide has never authorized a hide request.
|
|
8879
|
+
stopAttempted = true;
|
|
8880
|
+
exitConfirmed = ConfirmExit(delegate { return child.HasExited; }, child.Kill, child.WaitForExit);
|
|
8881
|
+
if (!exitConfirmed) throw new System.TimeoutException("guardian termination unconfirmed after 1000 ms");
|
|
8882
|
+
throw new System.TimeoutException("guardian startup timeout");
|
|
8883
|
+
}
|
|
8884
|
+
}
|
|
8885
|
+
exitConfirmed = true;
|
|
8886
|
+
lock (Gate)
|
|
8887
|
+
{
|
|
8888
|
+
s.Ready = false;
|
|
8889
|
+
restoreNeeded = s.Requested && view.ReadInt32(RestoredOffset) != 1;
|
|
8890
|
+
if (!s.Revoked) fault = "pointer guardian ended; native cursor fallback";
|
|
8891
|
+
}
|
|
8892
|
+
}
|
|
8893
|
+
catch (System.Exception e)
|
|
8894
|
+
{
|
|
8895
|
+
lock (Gate) { s.Ready = false; fault = "pointer guardian: " + e.Message; restoreNeeded = s.Requested; }
|
|
8896
|
+
// An alive guardian remains the sole hide writer. Never race an
|
|
8897
|
+
// emergency show against a process that can still issue hide.
|
|
8898
|
+
if (child != null && !exitConfirmed && !stopAttempted)
|
|
8899
|
+
{
|
|
8900
|
+
stopAttempted = true;
|
|
8901
|
+
exitConfirmed = ConfirmExit(delegate { return child.HasExited; }, child.Kill, child.WaitForExit);
|
|
8902
|
+
}
|
|
8903
|
+
if (!exitConfirmed)
|
|
8904
|
+
{
|
|
8905
|
+
restoreNeeded = false;
|
|
8906
|
+
if (child != null) lock (Gate) fault += "; guardian exit unconfirmed, emergency show suppressed and further leases disabled";
|
|
8907
|
+
}
|
|
8908
|
+
}
|
|
8909
|
+
finally
|
|
8910
|
+
{
|
|
8911
|
+
// Guardian has ended before emergency restoration and before a
|
|
8912
|
+
// subsequent session may start. Old cleanup cannot unhide a new lease.
|
|
8913
|
+
if (restoreNeeded && exitConfirmed) EmergencyShow();
|
|
8914
|
+
lock (Gate)
|
|
8915
|
+
{
|
|
8916
|
+
s.Ready = false; s.View = null; s.Map = null; s.Guardian = null;
|
|
8917
|
+
if (view != null) view.Dispose(); if (map != null) map.Dispose();
|
|
8918
|
+
if (child != null) child.Dispose(); s.Finished = true;
|
|
8919
|
+
}
|
|
8920
|
+
}
|
|
8921
|
+
}
|
|
8922
|
+
|
|
8923
|
+
// Bounded process shutdown, shared with fake process tests. Failure is
|
|
8924
|
+
// explicit: the caller must not race show against a possible late hide.
|
|
8925
|
+
internal static bool ConfirmExit(System.Func<bool> hasExited, System.Action kill, System.Func<int, bool> wait)
|
|
8926
|
+
{
|
|
8927
|
+
try
|
|
8928
|
+
{
|
|
8929
|
+
if (hasExited()) return true;
|
|
8930
|
+
try { kill(); } catch { return hasExited(); }
|
|
8931
|
+
return wait(1000) || hasExited();
|
|
8932
|
+
}
|
|
8933
|
+
catch { return false; }
|
|
8934
|
+
}
|
|
8935
|
+
|
|
8936
|
+
static void EmergencyShow()
|
|
8937
|
+
{
|
|
8938
|
+
var native = new NativeBackend(); bool initialized = false;
|
|
8939
|
+
try { initialized = native.Initialize(); if (!initialized || !native.Show(true)) lock (Gate) fault = "emergency cursor restore API failed"; }
|
|
8940
|
+
catch (System.Exception e) { lock (Gate) fault = "emergency cursor restore: " + e.Message; }
|
|
8941
|
+
finally { if (initialized) try { native.Uninitialize(); } catch { } }
|
|
8942
|
+
}
|
|
8943
|
+
|
|
8944
|
+
internal interface Backend { bool Initialize(); bool Show(bool visible); void Uninitialize(); }
|
|
8945
|
+
sealed class NativeBackend : Backend
|
|
8946
|
+
{
|
|
8947
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagInitialize();
|
|
8948
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagShowSystemCursor(bool show);
|
|
8949
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagUninitialize();
|
|
8950
|
+
public bool Initialize() { return MagInitialize(); }
|
|
8951
|
+
public bool Show(bool visible) { return MagShowSystemCursor(visible); }
|
|
8952
|
+
public void Uninitialize() { MagUninitialize(); }
|
|
8953
|
+
}
|
|
8954
|
+
internal sealed class Outcome
|
|
8955
|
+
{ public bool HideAttempted, RestoreConfirmed, Success; public string Reason; }
|
|
8956
|
+
|
|
8957
|
+
// Same loop is used in the actual guardian and deterministic fake-backend
|
|
8958
|
+
// tests. No wall-clock waiting or native API is required in those tests.
|
|
8959
|
+
internal static Outcome RunLease(Backend api, System.Func<Request> read,
|
|
8960
|
+
System.Func<long> now, System.Func<bool> ownerAlive, System.Action pause, System.Action ready)
|
|
8961
|
+
{
|
|
8962
|
+
var result = new Outcome(); bool initialized = false, hidden = false;
|
|
8963
|
+
long started = now(), latestSequence = 0, stamp = started; bool everRequested = false;
|
|
8964
|
+
try
|
|
8965
|
+
{
|
|
8966
|
+
initialized = api.Initialize();
|
|
8967
|
+
if (!initialized) { result.Reason = "initialize-failed"; return result; }
|
|
8968
|
+
ready();
|
|
8969
|
+
while (true)
|
|
8970
|
+
{
|
|
8971
|
+
if (!ownerAlive()) { result.Reason = "owner-exited"; break; }
|
|
8972
|
+
// Expiration is terminal even when a newer request has
|
|
8973
|
+
// arrived while this process was not scheduled.
|
|
8974
|
+
if (everRequested && now() - stamp >= LeaseMs) { result.Reason = "lease-expired"; break; }
|
|
8975
|
+
Request request = read(); long currentTime = now();
|
|
8976
|
+
if (request != null && request.Sequence > latestSequence)
|
|
8977
|
+
{
|
|
8978
|
+
latestSequence = request.Sequence;
|
|
8979
|
+
if (request.Revoked || (!request.Desired && everRequested)) { result.Reason = "released"; break; }
|
|
8980
|
+
if (request.Desired)
|
|
8981
|
+
{
|
|
8982
|
+
if (request.Stamp > currentTime || currentTime - request.Stamp >= LeaseMs)
|
|
8983
|
+
{ result.Reason = "expired-request"; break; }
|
|
8984
|
+
everRequested = true; stamp = request.Stamp;
|
|
8985
|
+
if (!hidden)
|
|
8986
|
+
{
|
|
8987
|
+
// Re-read the current complete request before the
|
|
8988
|
+
// sole hide writer acts; queued old commands never apply.
|
|
8989
|
+
Request newest = read();
|
|
8990
|
+
if (newest == null || newest.Sequence != request.Sequence) { pause(); continue; }
|
|
8991
|
+
if (!ownerAlive() || now() - stamp >= LeaseMs) { result.Reason = "expired-before-hide"; break; }
|
|
8992
|
+
result.HideAttempted = true;
|
|
8993
|
+
if (!api.Show(false)) { result.Reason = "hide-failed"; break; }
|
|
8994
|
+
hidden = true;
|
|
8995
|
+
// A release concurrent with the API is processed
|
|
8996
|
+
// before sleeping or accepting another lease.
|
|
8997
|
+
Request after = read();
|
|
8998
|
+
if (after != null && after.Sequence > latestSequence && (after.Revoked || !after.Desired))
|
|
8999
|
+
{ result.Reason = "released-during-hide"; break; }
|
|
9000
|
+
}
|
|
9001
|
+
}
|
|
9002
|
+
}
|
|
9003
|
+
if (everRequested && currentTime - stamp >= LeaseMs) { result.Reason = "lease-expired"; break; }
|
|
9004
|
+
if (!everRequested && currentTime - started >= StartupMs) { result.Reason = "unused"; break; }
|
|
9005
|
+
pause();
|
|
9006
|
+
}
|
|
9007
|
+
result.Success = result.Reason != "hide-failed";
|
|
9008
|
+
}
|
|
9009
|
+
catch (System.Exception e) { result.Reason = "fault: " + e.Message; }
|
|
9010
|
+
finally
|
|
9011
|
+
{
|
|
9012
|
+
if (initialized)
|
|
9013
|
+
{
|
|
9014
|
+
if (result.HideAttempted)
|
|
9015
|
+
{
|
|
9016
|
+
try { result.RestoreConfirmed = api.Show(true); }
|
|
9017
|
+
catch { result.RestoreConfirmed = false; }
|
|
9018
|
+
if (!result.RestoreConfirmed) result.Success = false;
|
|
9019
|
+
}
|
|
9020
|
+
try { api.Uninitialize(); } catch { result.Success = false; }
|
|
9021
|
+
}
|
|
9022
|
+
}
|
|
9023
|
+
return result;
|
|
9024
|
+
}
|
|
9025
|
+
|
|
9026
|
+
// Dispatch this at the FIRST line of worker Main, before Panic.Init,
|
|
9027
|
+
// hooks, global brake, protocol stdout, or any other worker lifecycle.
|
|
9028
|
+
public static int GuardianMain(string name, int ownerPid, long ownerStartTicks)
|
|
9029
|
+
{
|
|
9030
|
+
if (System.IntPtr.Size != 8 || name == null || !name.StartsWith(Prefix, System.StringComparison.Ordinal) ||
|
|
9031
|
+
name.Length != Prefix.Length + 32) return 2;
|
|
9032
|
+
try
|
|
9033
|
+
{
|
|
9034
|
+
using (System.Diagnostics.Process owner = System.Diagnostics.Process.GetProcessById(ownerPid))
|
|
9035
|
+
using (System.Diagnostics.Process self = System.Diagnostics.Process.GetCurrentProcess())
|
|
9036
|
+
using (var map = System.IO.MemoryMappedFiles.MemoryMappedFile.OpenExisting(name))
|
|
9037
|
+
using (var view = map.CreateViewAccessor(0, Bytes))
|
|
9038
|
+
{
|
|
9039
|
+
if (owner.StartTime.ToUniversalTime().Ticks != ownerStartTicks || owner.SessionId != self.SessionId ||
|
|
9040
|
+
view.ReadInt32(0) != Version || view.ReadInt32(4) != ownerPid || view.ReadInt64(8) != ownerStartTicks) return 3;
|
|
9041
|
+
Outcome result = RunLease(new NativeBackend(), delegate { return ReadRequest(view); }, NowMs,
|
|
9042
|
+
delegate { try { return !owner.HasExited; } catch { return false; } },
|
|
9043
|
+
delegate { System.Threading.Thread.Sleep(20); },
|
|
9044
|
+
delegate { view.Write(ReadyOffset, 1); });
|
|
9045
|
+
view.Write(RestoredOffset, result.RestoreConfirmed ? 1 : 0);
|
|
9046
|
+
System.Threading.Thread.MemoryBarrier(); view.Write(ReadyOffset, result.Success ? 2 : 3);
|
|
9047
|
+
return result.Success ? 0 : 4;
|
|
9048
|
+
}
|
|
9049
|
+
}
|
|
9050
|
+
catch { return 5; }
|
|
9051
|
+
}
|
|
9052
|
+
}
|
|
9053
|
+
}
|