dsh-codex-style-computer-use 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -22
- package/README.zh.md +46 -21
- package/icon.svg +29 -0
- package/lib/audit.js +7 -0
- package/lib/batch.js +14 -19
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/compact-feedback.js +175 -0
- package/lib/continuation.js +1 -1
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/compile-worker.ps1 +5 -1
- package/lib/core/index.js +452 -129
- package/lib/core/native-runtime.js +23 -1
- package/lib/core/winmd/Windows.winmd +0 -0
- package/lib/core/worker.cs +2415 -356
- package/lib/feedback-routes.json +11 -0
- package/lib/feedback.js +49 -16
- package/lib/image-output.js +59 -11
- package/lib/index.js +42 -7
- package/lib/key-command.js +37 -0
- package/lib/media-contract.js +43 -0
- package/lib/observations.js +267 -13
- package/lib/policy.js +30 -10
- package/lib/tool-contract.js +56 -0
- package/lib/tools.js +174 -51
- package/lib/verification-region.js +17 -0
- package/mcp/package.json +2 -1
- package/mcp/src/execution.js +8 -5
- package/mcp/src/index.js +16 -267
- package/mcp/src/tools.js +271 -0
- package/package.json +5 -2
- package/scripts/check-behavior.mjs +83 -0
- package/scripts/measure-tool-tokens.mjs +21 -0
- package/scripts/render-promo.py +81 -0
- package/skills/computer-use/SKILL.md +35 -8
- package/skills/computer-use/references/lifecycle.md +11 -7
- package/skills/computer-use/references/maintenance.md +9 -5
- package/skills/computer-use/references/precision.md +39 -14
- package/skills/computer-use/references/progressive-control.md +10 -4
- package/skills/computer-use/references/zero-uia-apps.md +54 -0
package/lib/core/worker.cs
CHANGED
|
@@ -20,31 +20,51 @@ using System.Collections.Generic;
|
|
|
20
20
|
using System.Collections.Concurrent;
|
|
21
21
|
using System.Diagnostics;
|
|
22
22
|
using System.Drawing;
|
|
23
|
+
using System.Drawing.Drawing2D;
|
|
23
24
|
using System.Drawing.Imaging;
|
|
24
25
|
using System.IO;
|
|
25
26
|
using System.Linq;
|
|
26
27
|
using System.Runtime.InteropServices;
|
|
28
|
+
using System.Runtime.InteropServices.WindowsRuntime; // AsBuffer: byte[] -> WinRT IBuffer
|
|
27
29
|
using System.Text;
|
|
28
30
|
using System.Threading;
|
|
29
31
|
using System.Web.Script.Serialization;
|
|
30
32
|
using System.Windows.Automation;
|
|
31
33
|
using System.Windows.Forms; // Clipboard (watch out: bare Point stays our own struct)
|
|
34
|
+
// Inbox OCR (zero external dependencies): the union Windows.winmd ships in lib/core/winmd.
|
|
35
|
+
using Windows.Media.Ocr;
|
|
36
|
+
using Windows.Graphics.Imaging;
|
|
32
37
|
|
|
33
38
|
namespace DshComputerUse.Worker
|
|
34
39
|
{
|
|
35
|
-
//
|
|
40
|
+
// On-demand cadence: one monotonic timestamp, no timer, history list or disk writes.
|
|
41
|
+
// Only a successfully displayed cue advances the sliding three-minute window.
|
|
42
|
+
public sealed class PreparationCadence
|
|
43
|
+
{
|
|
44
|
+
public const int FullMs = 1000, RepeatMs = 100, RepeatWindowMs = 180000;
|
|
45
|
+
long lastShown = -1;
|
|
46
|
+
public int Duration(long now)
|
|
47
|
+
{ long previous = Interlocked.Read(ref lastShown); return previous >= 0 && now >= previous && now - previous < RepeatWindowMs ? RepeatMs : FullMs; }
|
|
48
|
+
public void Presented(long now) { Interlocked.Exchange(ref lastShown, now); }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// A live cycle observes physical holds; actuation or an explicit scene hold can start a handoff.
|
|
36
52
|
// Once yielded, three quiet seconds with no held input returns control to the agent.
|
|
37
53
|
public sealed class HumanInputState
|
|
38
54
|
{
|
|
39
55
|
public const int SustainedMs = 2000, IdleMs = 3000;
|
|
40
56
|
readonly object sync = new object();
|
|
41
57
|
readonly HashSet<int> held = new HashSet<int>();
|
|
58
|
+
readonly Dictionary<int, int> releasedSamples = new Dictionary<int, int>();
|
|
59
|
+
readonly HashSet<int> injectedTouched = new HashSet<int>();
|
|
42
60
|
bool enabled, acting, seen;
|
|
43
61
|
string phase = "idle", lastKind = "none";
|
|
44
|
-
long first, last, revision, burst;
|
|
62
|
+
long first, last, revision, burst, stateSeq;
|
|
63
|
+
int reconciledReleases;
|
|
45
64
|
public string Phase { get { lock (sync) { return phase; } } }
|
|
46
65
|
public long Revision { get { lock (sync) { return revision; } } }
|
|
47
66
|
public bool Active { get { lock (sync) { return phase != "idle"; } } }
|
|
67
|
+
public bool HasHeld { get { lock (sync) { return held.Count != 0; } } }
|
|
48
68
|
public string LastKind { get { lock (sync) { return lastKind; } } }
|
|
49
69
|
public bool Held(int vk)
|
|
50
70
|
{
|
|
@@ -65,7 +85,7 @@ namespace DshComputerUse.Worker
|
|
|
65
85
|
lock (sync)
|
|
66
86
|
{
|
|
67
87
|
if (enabled == on) return;
|
|
68
|
-
enabled = on;
|
|
88
|
+
enabled = on; stateSeq++;
|
|
69
89
|
if (!on) { phase = "idle"; acting = false; }
|
|
70
90
|
}
|
|
71
91
|
}
|
|
@@ -73,21 +93,28 @@ namespace DshComputerUse.Worker
|
|
|
73
93
|
{
|
|
74
94
|
lock (sync)
|
|
75
95
|
{
|
|
76
|
-
acting = on;
|
|
96
|
+
if (acting != on) { acting = on; stateSeq++; }
|
|
77
97
|
// Do not replay earlier ordinary activity. A key/button still physically
|
|
78
98
|
// held does need ownership before we can inject a conflicting chord.
|
|
79
99
|
if (on && enabled && phase == "idle" && held.Count > 0)
|
|
80
|
-
{ last = now; revision++; Start(now); }
|
|
100
|
+
{ last = now; revision++; stateSeq++; Start(now); }
|
|
81
101
|
}
|
|
82
102
|
}
|
|
83
103
|
void Start(long now) { phase = "yielding"; first = now; if (!seen) last = now; burst++; }
|
|
84
|
-
public void Record(int key, bool down, bool up, long now)
|
|
104
|
+
public void Record(int key, bool down, bool up, long now) { Record(key, down, up, now, false); }
|
|
105
|
+
/// Pointer jitter is filtered before this call. The protection flag includes both
|
|
106
|
+
/// actual injection and a retained scene hold; free observation never takes ownership.
|
|
107
|
+
public void Record(int key, bool down, bool up, long now, bool deliberate)
|
|
85
108
|
{
|
|
86
109
|
lock (sync)
|
|
87
110
|
{
|
|
88
111
|
bool continued = held.Count > 0 || (seen && now - last < IdleMs);
|
|
112
|
+
stateSeq++;
|
|
89
113
|
lastKind = key == 0 ? "pointer-or-wheel" : key <= 6 ? "button" : "keyboard";
|
|
90
|
-
if (key != 0) {
|
|
114
|
+
if (key != 0) {
|
|
115
|
+
if (down) held.Add(key); if (up) held.Remove(key);
|
|
116
|
+
releasedSamples.Remove(key); injectedTouched.Remove(key);
|
|
117
|
+
}
|
|
91
118
|
seen = true; last = now;
|
|
92
119
|
if (!enabled || (phase == "idle" && !acting)) return;
|
|
93
120
|
revision++;
|
|
@@ -100,19 +127,110 @@ namespace DshComputerUse.Worker
|
|
|
100
127
|
lock (sync)
|
|
101
128
|
{
|
|
102
129
|
if (!enabled || phase == "idle") return;
|
|
103
|
-
if (held.Count == 0 && now - last >= IdleMs) { phase = "idle"; return; }
|
|
104
|
-
if (held.Count > 0 && now - first >= SustainedMs) phase = "waiting";
|
|
130
|
+
if (held.Count == 0 && now - last >= IdleMs) { phase = "idle"; stateSeq++; return; }
|
|
131
|
+
if (held.Count > 0 && now - first >= SustainedMs && phase != "waiting") { phase = "waiting"; stateSeq++; }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
public void NoteInjected(int key)
|
|
135
|
+
{
|
|
136
|
+
// An injected UP can make async state look released while a human
|
|
137
|
+
// still physically holds the same key. Never reconcile that case.
|
|
138
|
+
lock (sync) {
|
|
139
|
+
foreach (int tracked in held)
|
|
140
|
+
if (tracked == key || ModifierFamily(tracked) != 0 && ModifierFamily(tracked) == ModifierFamily(key) ||
|
|
141
|
+
(tracked == 1 || tracked == 2) && (key == 1 || key == 2))
|
|
142
|
+
{ injectedTouched.Add(tracked); releasedSamples.Remove(tracked); }
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
static int ModifierFamily(int key)
|
|
146
|
+
{ return key == 0x10 || key == 0xA0 || key == 0xA1 ? 0x10 : key == 0x11 || key == 0xA2 || key == 0xA3 ? 0x11 : key == 0x12 || key == 0xA4 || key == 0xA5 ? 0x12 : 0; }
|
|
147
|
+
public static bool AsyncButtonMayBeDown(int key, bool swapped, Func<int, bool> isDown)
|
|
148
|
+
{
|
|
149
|
+
// Async state names PHYSICAL buttons. With swapped primary/secondary
|
|
150
|
+
// semantics, require BOTH physical buttons up before repairing either
|
|
151
|
+
// history entry; never infer the low-level hook's mapping from a UI label.
|
|
152
|
+
return swapped && (key == 1 || key == 2) ? isDown(1) || isDown(2) : isDown(key);
|
|
153
|
+
}
|
|
154
|
+
public int ReconcileReleased(Func<int, bool> isDown, bool trusted, long now)
|
|
155
|
+
{
|
|
156
|
+
lock (sync) {
|
|
157
|
+
if (!trusted) { releasedSamples.Clear(); return 0; }
|
|
158
|
+
int removed = 0;
|
|
159
|
+
foreach (int key in held.ToArray()) {
|
|
160
|
+
if (injectedTouched.Contains(key) || isDown(key)) { releasedSamples.Remove(key); continue; }
|
|
161
|
+
int count; releasedSamples.TryGetValue(key, out count);
|
|
162
|
+
if (++count < 3) { releasedSamples[key] = count; continue; }
|
|
163
|
+
held.Remove(key); releasedSamples.Remove(key); removed++;
|
|
164
|
+
}
|
|
165
|
+
if (removed > 0) {
|
|
166
|
+
// This is fresh release evidence, not permission from elapsed
|
|
167
|
+
// time: still require the full quiet interval after this point.
|
|
168
|
+
last = now; seen = true; lastKind = "reconciled-release";
|
|
169
|
+
reconciledReleases += removed; revision++; stateSeq++;
|
|
170
|
+
}
|
|
171
|
+
return removed;
|
|
105
172
|
}
|
|
106
173
|
}
|
|
107
174
|
public object Snapshot(long now)
|
|
108
175
|
{
|
|
109
176
|
lock (sync) return Program.Dict("phase", phase, "active", phase != "idle",
|
|
110
|
-
"revision", revision, "burst", burst, "heldCount", held.Count, "lastKind", lastKind,
|
|
177
|
+
"revision", revision, "stateSeq", stateSeq, "burst", burst, "heldCount", held.Count, "lastKind", lastKind,
|
|
178
|
+
"reconciledReleaseCount", reconciledReleases,
|
|
111
179
|
"quietRemainingMs", phase == "idle" ? 0 : held.Count > 0 ? IdleMs : Math.Max(0, IdleMs - (now - last)),
|
|
112
180
|
"sustainedMs", SustainedMs, "idleMs", IdleMs);
|
|
113
181
|
}
|
|
114
182
|
}
|
|
115
183
|
|
|
184
|
+
// Domain ownership survives renderer failures and visibility changes. Glow
|
|
185
|
+
// only reads this state; cycle/operation lifecycle explicitly changes it.
|
|
186
|
+
public static class SceneControl
|
|
187
|
+
{
|
|
188
|
+
static volatile bool acting, holding;
|
|
189
|
+
public static bool Acting { get { return acting; } }
|
|
190
|
+
public static bool Holding { get { return holding; } }
|
|
191
|
+
public static void SetHolding(bool on)
|
|
192
|
+
{
|
|
193
|
+
holding = on && Panic.CycleLit && !Panic.Exited && !Panic.Stopped && !Panic.Engaged;
|
|
194
|
+
Panic.UpdateInputProtection();
|
|
195
|
+
}
|
|
196
|
+
public static void BeginActuation() { acting = true; SetHolding(true); }
|
|
197
|
+
public static void EndActuation() { acting = false; Panic.UpdateInputProtection(); }
|
|
198
|
+
public static void Reset() { acting = holding = false; Panic.UpdateInputProtection(); }
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// The UAC secure desktop (and the lock screen) is the ONE surface automation must never
|
|
202
|
+
// reach — Windows enforces that itself. The honest addition (human request 2026-11-04)
|
|
203
|
+
// is to NOTICE the switch, refuse to dispatch while it stands, tell the human exactly
|
|
204
|
+
// what to do, and auto-continue the moment the normal desktop is back. Detection is an
|
|
205
|
+
// OpenInputDesktop probe: while the secure desktop is up, an unprivileged thread cannot
|
|
206
|
+
// open the input desktop at all. Transitions are confirmed over consecutive misses so a
|
|
207
|
+
// single hiccup never pauses dispatch, and one success clears the state immediately.
|
|
208
|
+
public sealed class SecureDesktopWatch
|
|
209
|
+
{
|
|
210
|
+
public const int ConfirmMisses = 3;
|
|
211
|
+
readonly object sync = new object();
|
|
212
|
+
bool active; int misses;
|
|
213
|
+
public bool Active { get { lock (sync) { return active; } } }
|
|
214
|
+
/// Returns true only on a STATE TRANSITION; the caller audits and notifies those alone.
|
|
215
|
+
public bool Observe(bool reachable)
|
|
216
|
+
{
|
|
217
|
+
lock (sync)
|
|
218
|
+
{
|
|
219
|
+
if (reachable)
|
|
220
|
+
{
|
|
221
|
+
misses = 0;
|
|
222
|
+
if (!active) return false;
|
|
223
|
+
active = false;
|
|
224
|
+
return true; // secured -> normal: dispatch may resume
|
|
225
|
+
}
|
|
226
|
+
if (active) return false;
|
|
227
|
+
if (++misses < ConfirmMisses) return false;
|
|
228
|
+
active = true;
|
|
229
|
+
return true; // normal -> secured
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
116
234
|
// Sum only physical displacement, rebasing every injected cursor event. Agent
|
|
117
235
|
// motion cannot turn a 1-pixel hand tremor into a takeover. Back-and-forth jitter
|
|
118
236
|
// cancels rather than accumulating path length; a deliberate 16-pixel move yields.
|
|
@@ -216,7 +334,12 @@ namespace DshComputerUse.Worker
|
|
|
216
334
|
if (!active || paused || exited) return null;
|
|
217
335
|
lock (sync)
|
|
218
336
|
{
|
|
219
|
-
if (hwnd == 0 || pid == 0)
|
|
337
|
+
if (hwnd == 0 || pid == 0)
|
|
338
|
+
{
|
|
339
|
+
const string missing = "this worker has no confirmed target for the current cycle";
|
|
340
|
+
if (host || foreground == 0 || process == 0) return missing;
|
|
341
|
+
return missing + ". Observed foreground hwnd=" + foreground + ", pid=" + process;
|
|
342
|
+
}
|
|
220
343
|
if (!targetExists) return "the activated target window no longer exists";
|
|
221
344
|
if (host) return "the agent host is in front of the target window";
|
|
222
345
|
if (hwnd != foreground || pid != process) return "the foreground no longer matches the activated target";
|
|
@@ -398,6 +521,10 @@ namespace DshComputerUse.Worker
|
|
|
398
521
|
public static extern bool OpenProcessToken(IntPtr proc, uint access, out IntPtr token);
|
|
399
522
|
[DllImport("advapi32.dll", SetLastError = true)]
|
|
400
523
|
public static extern bool GetTokenInformation(IntPtr token, int cls, out uint info, uint len, out uint ret);
|
|
524
|
+
[DllImport("advapi32.dll", EntryPoint = "GetTokenInformation", SetLastError = true)]
|
|
525
|
+
public static extern bool GetTokenInformationBuffer(IntPtr token, int cls, IntPtr info, uint len, out uint ret);
|
|
526
|
+
[DllImport("advapi32.dll")] public static extern IntPtr GetSidSubAuthorityCount(IntPtr sid);
|
|
527
|
+
[DllImport("advapi32.dll")] public static extern IntPtr GetSidSubAuthority(IntPtr sid, uint index);
|
|
401
528
|
public const uint PROCESS_QUERY_LIMITED_INFORMATION = 0x1000, TOKEN_QUERY = 0x0008;
|
|
402
529
|
public const int TokenElevation = 20;
|
|
403
530
|
[DllImport("user32.dll")]
|
|
@@ -424,10 +551,23 @@ namespace DshComputerUse.Worker
|
|
|
424
551
|
public static extern IntPtr CreateDIBSection(IntPtr dc, ref BITMAPINFO bmi, uint usage, out IntPtr bits, IntPtr hSection, uint offset);
|
|
425
552
|
[DllImport("user32.dll")]
|
|
426
553
|
public static extern bool GetCursorInfo(ref CURSORINFO pci);
|
|
554
|
+
[DllImport("user32.dll", EntryPoint = "LoadCursorW")]
|
|
555
|
+
public static extern IntPtr LoadCursor(IntPtr instance, IntPtr resource);
|
|
427
556
|
[DllImport("user32.dll")]
|
|
428
557
|
public static extern bool DrawIconEx(IntPtr hdc, int xLeft, int yTop, IntPtr hIcon, int cxWidth, int cyHeight, int istepIfAniCur, IntPtr hbrFlickerFreeDraw, uint diFlags);
|
|
429
558
|
[DllImport("user32.dll")]
|
|
430
559
|
public static extern uint SendNotifyMessage(IntPtr hWnd, uint msg, IntPtr w, IntPtr l);
|
|
560
|
+
// Secure-desktop probe: while the UAC prompt (or the lock screen) is up, an unprivileged
|
|
561
|
+
// thread CANNOT open the input desktop — that refusal is the only reliable, permission-free
|
|
562
|
+
// witness that the machine switched to the surface automation must never touch.
|
|
563
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
564
|
+
public static extern IntPtr OpenInputDesktop(uint flags, bool inherit, uint access);
|
|
565
|
+
[DllImport("user32.dll")]
|
|
566
|
+
public static extern bool CloseDesktop(IntPtr h);
|
|
567
|
+
[DllImport("user32.dll")]
|
|
568
|
+
public static extern IntPtr GetThreadDesktop(uint threadId);
|
|
569
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
|
570
|
+
public static extern bool GetUserObjectInformation(IntPtr obj, int index, StringBuilder info, int length, out int needed);
|
|
431
571
|
|
|
432
572
|
// ---- low-level keyboard hook: the human's emergency brake ----
|
|
433
573
|
public delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);
|
|
@@ -454,7 +594,7 @@ namespace DshComputerUse.Worker
|
|
|
454
594
|
[StructLayout(LayoutKind.Sequential)]
|
|
455
595
|
public struct KBDLLHOOKSTRUCT { public uint vkCode, scanCode, flags, time; public IntPtr dwExtraInfo; }
|
|
456
596
|
[StructLayout(LayoutKind.Sequential)]
|
|
457
|
-
public struct NativeMsg { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; }
|
|
597
|
+
public struct NativeMsg { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; public uint lPrivate; }
|
|
458
598
|
public const int WH_KEYBOARD_LL = 13, WM_KEYDOWN_ = 0x0100, WM_SYSKEYDOWN_ = 0x0104,
|
|
459
599
|
WM_KEYUP_ = 0x0101, WM_SYSKEYUP_ = 0x0105;
|
|
460
600
|
// KBDLLHOOKSTRUCT.flags bit 4: the key did NOT come from the physical keyboard — some
|
|
@@ -494,7 +634,6 @@ namespace DshComputerUse.Worker
|
|
|
494
634
|
[StructLayout(LayoutKind.Sequential)] struct SIZEL { public int cx, cy; }
|
|
495
635
|
[StructLayout(LayoutKind.Sequential, Pack = 1)]
|
|
496
636
|
struct BLENDFUNCTION { public byte BlendOp, BlendFlags, SourceConstantAlpha, AlphaFormat; }
|
|
497
|
-
[StructLayout(LayoutKind.Sequential)] struct MSG { public IntPtr hwnd; public uint message; public IntPtr wParam, lParam; public uint time; public int ptX, ptY; }
|
|
498
637
|
|
|
499
638
|
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
500
639
|
static extern IntPtr CreateWindowEx(int exStyle, string cls, string name, int style,
|
|
@@ -504,23 +643,47 @@ namespace DshComputerUse.Worker
|
|
|
504
643
|
[DllImport("user32.dll")] static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int cx, int cy, uint flags);
|
|
505
644
|
[DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr h);
|
|
506
645
|
[DllImport("user32.dll")] static extern int ReleaseDC(IntPtr h, IntPtr dc);
|
|
507
|
-
[DllImport("user32.dll")] static extern bool PeekMessage(out
|
|
646
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern bool PeekMessage(out Native.NativeMsg m, IntPtr h, uint min, uint max, uint remove);
|
|
647
|
+
[DllImport("user32.dll")] static extern bool ValidateRect(IntPtr h, IntPtr rect);
|
|
508
648
|
[DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr dc);
|
|
509
649
|
[DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr dc);
|
|
510
650
|
[DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr dc, IntPtr o);
|
|
511
651
|
[DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr o);
|
|
512
652
|
[DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr dc, ref Native.BITMAPINFO bmi, uint usage, out IntPtr bits, IntPtr section, uint offset);
|
|
513
|
-
[DllImport("user32.dll")] static extern bool UpdateLayeredWindow(IntPtr h, IntPtr dst, ref POINTL pptDst, ref SIZEL psize,
|
|
653
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool UpdateLayeredWindow(IntPtr h, IntPtr dst, ref POINTL pptDst, ref SIZEL psize,
|
|
514
654
|
IntPtr src, ref POINTL pptSrc, int colorKey, ref BLENDFUNCTION blend, int flags);
|
|
515
655
|
[DllImport("user32.dll")] static extern int GetSystemMetrics(int i);
|
|
516
656
|
[DllImport("user32.dll")] static extern bool SetWindowDisplayAffinity(IntPtr hwnd, uint affinity);
|
|
517
657
|
|
|
518
658
|
static Thread _thread;
|
|
519
|
-
static volatile bool _acting;
|
|
520
659
|
static volatile bool _preparing;
|
|
521
660
|
static volatile int _preparationStart;
|
|
522
|
-
static int _preparationSerial, _presentedPreparation;
|
|
523
|
-
|
|
661
|
+
static int _preparationSerial, _presentedPreparation, _completedPreparation;
|
|
662
|
+
static volatile string _rendererStage = "not-started", _rendererFault;
|
|
663
|
+
static volatile int _lastLoopTick, _lastPresentTick, _lastFailedError;
|
|
664
|
+
static int _frames, _presentAttempts, _presentSuccesses, _lastAttemptWant, _lastAttemptSerial;
|
|
665
|
+
static int _lastAttemptedPreparationSerial = -1, _lastFailedSerial = -1, _lastFailedWant, _rebuiltSerial = -1;
|
|
666
|
+
public static Dictionary<string, object> Health()
|
|
667
|
+
{
|
|
668
|
+
var thread = _thread;
|
|
669
|
+
int now = Environment.TickCount;
|
|
670
|
+
return Program.Dict("workerPid", System.Diagnostics.Process.GetCurrentProcess().Id,
|
|
671
|
+
"threadAlive", thread != null && thread.IsAlive, "stage", _rendererStage, "fault", _rendererFault,
|
|
672
|
+
"want", Want, "preparationSerial", Volatile.Read(ref _preparationSerial), "presentedSerial", Volatile.Read(ref _presentedPreparation),
|
|
673
|
+
"completedSerial", Volatile.Read(ref _completedPreparation),
|
|
674
|
+
"loopAgeMs", Volatile.Read(ref _frames) == 0 ? (object)null : unchecked((uint)(now - _lastLoopTick)),
|
|
675
|
+
"presentAttempts", Volatile.Read(ref _presentAttempts), "lastAttemptWant", Volatile.Read(ref _lastAttemptWant),
|
|
676
|
+
"lastAttemptSerial", Volatile.Read(ref _lastAttemptSerial), "presentSuccesses", Volatile.Read(ref _presentSuccesses),
|
|
677
|
+
"lastAttemptedPreparationSerial", Volatile.Read(ref _lastAttemptedPreparationSerial),
|
|
678
|
+
"lastSuccessAgeMs", Volatile.Read(ref _presentSuccesses) == 0 ? (object)null : unchecked((uint)(now - _lastPresentTick)),
|
|
679
|
+
"lastFailedSerial", Volatile.Read(ref _lastFailedSerial), "lastFailedWant", Volatile.Read(ref _lastFailedWant),
|
|
680
|
+
"lastWin32Error", Volatile.Read(ref _lastFailedSerial) < 0 ? (object)null : _lastFailedError,
|
|
681
|
+
"rebuiltSerial", Volatile.Read(ref _rebuiltSerial), "pointerFault", AgentPointer.Fault);
|
|
682
|
+
}
|
|
683
|
+
static readonly Stopwatch PreparationClock = Stopwatch.StartNew();
|
|
684
|
+
static readonly PreparationCadence PreparationFrequency = new PreparationCadence();
|
|
685
|
+
static volatile int _preparationMs = PreparationCadence.FullMs;
|
|
686
|
+
public static int PreparationMs { get { return _preparationMs; } }
|
|
524
687
|
static volatile bool _enabled = true;
|
|
525
688
|
static volatile string _label = "AI 正在操控此电脑 · 监听中";
|
|
526
689
|
|
|
@@ -528,21 +691,36 @@ namespace DshComputerUse.Worker
|
|
|
528
691
|
public static void Touch(int holdMs)
|
|
529
692
|
{
|
|
530
693
|
if (!_enabled) return;
|
|
694
|
+
_sceneVisible = true;
|
|
531
695
|
_thinking = true;
|
|
532
696
|
EnsureStarted();
|
|
533
697
|
}
|
|
534
|
-
|
|
535
|
-
|
|
698
|
+
// Renderer visibility is deliberately separate from scene ownership.
|
|
699
|
+
// Hiding the indicator must neither release the scene nor resurrect it
|
|
700
|
+
// on the next frame solely because the authoritative hold is still true.
|
|
701
|
+
static volatile bool _sceneVisible = true;
|
|
702
|
+
public static void Off() { _sceneVisible = false; _preparing = false; _thinking = false; }
|
|
703
|
+
public static bool Acting { get { return SceneControl.Acting; } }
|
|
704
|
+
public static bool Holding { get { return SceneControl.Holding; } }
|
|
536
705
|
public static bool Preparing { get { return _preparing; } }
|
|
537
706
|
public static bool PreparationVisible { get { return _preparing && Volatile.Read(ref _presentedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
538
|
-
public static
|
|
539
|
-
public static void
|
|
707
|
+
public static bool PreparationComplete { get { return _preparing && Volatile.Read(ref _completedPreparation) == Volatile.Read(ref _preparationSerial); } }
|
|
708
|
+
public static void ActuationBegin() { _sceneVisible = true; EnsureStarted(); }
|
|
709
|
+
public static void ActuationEnd() { }
|
|
540
710
|
public static void Prepare(bool on)
|
|
541
711
|
{
|
|
542
712
|
if (!on) { _preparing = false; return; }
|
|
543
713
|
_preparationStart = 0;
|
|
714
|
+
_preparationMs = PreparationFrequency.Duration(PreparationClock.ElapsedMilliseconds);
|
|
544
715
|
Interlocked.Increment(ref _preparationSerial);
|
|
545
|
-
_preparing = true; _enabled = true; EnsureStarted();
|
|
716
|
+
_preparing = true; _enabled = true; _sceneVisible = true; EnsureStarted();
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
public static void PreparationPresented(int serial)
|
|
720
|
+
{
|
|
721
|
+
_preparationStart = Environment.TickCount;
|
|
722
|
+
PreparationFrequency.Presented(PreparationClock.ElapsedMilliseconds);
|
|
723
|
+
Volatile.Write(ref _presentedPreparation, serial);
|
|
546
724
|
}
|
|
547
725
|
|
|
548
726
|
// Nested injection sequences stay observable across drag/key/typing gaps.
|
|
@@ -551,7 +729,7 @@ namespace DshComputerUse.Worker
|
|
|
551
729
|
public static void InputBegin() { Interlocked.Increment(ref _injecting); }
|
|
552
730
|
public static void InputEnd() { if (Interlocked.Decrement(ref _injecting) < 0) _injecting = 0; }
|
|
553
731
|
public static bool Injecting { get { return _injecting > 0; } }
|
|
554
|
-
public static void KeepActing() { /*
|
|
732
|
+
public static void KeepActing() { /* Solid blue follows actual input; the separate scene hold survives it. */ }
|
|
555
733
|
|
|
556
734
|
// Emergency stop must LOOK like one. The normal fade is a slow ease (~1.6 s) which,
|
|
557
735
|
// during a panic, reads as "nothing happened" — so this snaps the overlay to zero on
|
|
@@ -559,7 +737,7 @@ namespace DshComputerUse.Worker
|
|
|
559
737
|
static volatile bool _killNow = false;
|
|
560
738
|
public static void Kill()
|
|
561
739
|
{
|
|
562
|
-
|
|
740
|
+
_sceneVisible = false; _preparing = false; _thinking = false; _flashStart = 0;
|
|
563
741
|
_killNow = true;
|
|
564
742
|
}
|
|
565
743
|
|
|
@@ -575,6 +753,7 @@ namespace DshComputerUse.Worker
|
|
|
575
753
|
public static void Thinking(int holdMs)
|
|
576
754
|
{
|
|
577
755
|
if (!_enabled) return;
|
|
756
|
+
_sceneVisible = true;
|
|
578
757
|
_thinking = true;
|
|
579
758
|
EnsureStarted();
|
|
580
759
|
}
|
|
@@ -611,7 +790,8 @@ namespace DshComputerUse.Worker
|
|
|
611
790
|
public static int DeriveWant(bool flashOn, int committed)
|
|
612
791
|
{
|
|
613
792
|
if (Panic.Exited) return 0;
|
|
614
|
-
|
|
793
|
+
bool sceneVisible = _enabled && _sceneVisible;
|
|
794
|
+
return _askMode ? 5 : (_stoppedMode ? 4 : (Panic.HumanActive ? 6 : (_preparing ? 7 : (Acting && sceneVisible ? 1 : (Holding && sceneVisible ? 8 : (flashOn ? 2 : committed))))));
|
|
615
795
|
}
|
|
616
796
|
/// The title states the same fact as the light, derived the same way (an ended session must
|
|
617
797
|
/// not advertise "已暂停 · Ctrl+Alt+R 继续" in the window list).
|
|
@@ -619,7 +799,7 @@ namespace DshComputerUse.Worker
|
|
|
619
799
|
{
|
|
620
800
|
if (Panic.Exited) return "AI 已停止操控 · 未监听";
|
|
621
801
|
return _askMode ? "AI 正在提问 · 在 DSH 聊天里回答 · 无倒计时(Ctrl+Alt+R 撤销)"
|
|
622
|
-
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 ·
|
|
802
|
+
: (_stoppedMode ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : Panic.HumanActive ? "已让出键鼠 · 等待人操作 · 静默 3 秒后继续" : _preparing ? "即将开始操作,请暂停键鼠 · " + (PreparationMs / 1000.0).ToString("0.0") + " 秒准备" : Acting ? "AI 正在操作 · 请暂勿移动键鼠" : Holding ? "AI 正在检查 · 请保持页面与鼠标位置" : ThinkingActive ? "可自由使用键鼠 · AI 未占用现场" : label);
|
|
623
803
|
}
|
|
624
804
|
static volatile string _shownTitle = null;
|
|
625
805
|
// What the indicator is ACTUALLY rendering right now, readable from outside via `probe`.
|
|
@@ -633,14 +813,9 @@ namespace DshComputerUse.Worker
|
|
|
633
813
|
static extern bool SetWindowText(IntPtr h, string text);
|
|
634
814
|
public static void Stopped(bool on)
|
|
635
815
|
{
|
|
816
|
+
// Only the window owner sets the title (in Run). A physical brake must
|
|
817
|
+
// never wait for WM_SETTEXT to be serviced by the overlay thread.
|
|
636
818
|
_stoppedMode = on;
|
|
637
|
-
// The overlay's WINDOW TITLE is part of the text channel: renaming it means any
|
|
638
|
-
// window list — the agent's own computer_windows, or a human's Task Manager —
|
|
639
|
-
// states plainly that the session is stopped, with no screenshot required.
|
|
640
|
-
if (_hwnd != IntPtr.Zero)
|
|
641
|
-
{
|
|
642
|
-
try { SetWindowText(_hwnd, on ? "已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出" : _label); } catch { }
|
|
643
|
-
}
|
|
644
819
|
}
|
|
645
820
|
|
|
646
821
|
/// Mark "the agent just captured the screen": a yellow border flashes around the
|
|
@@ -690,7 +865,7 @@ namespace DshComputerUse.Worker
|
|
|
690
865
|
|
|
691
866
|
/// Force the overlay into existence. Used when a stop is ADOPTED from another process:
|
|
692
867
|
/// the red border must be visible even though this process never actuated anything.
|
|
693
|
-
public static void Reveal() { _enabled = true; EnsureStarted(); }
|
|
868
|
+
public static void Reveal() { _enabled = true; _sceneVisible = true; EnsureStarted(); }
|
|
694
869
|
|
|
695
870
|
static void EnsureStarted()
|
|
696
871
|
{
|
|
@@ -707,38 +882,91 @@ namespace DshComputerUse.Worker
|
|
|
707
882
|
|
|
708
883
|
static void ThreadProc()
|
|
709
884
|
{
|
|
710
|
-
try { Run(); }
|
|
885
|
+
try { Run(); }
|
|
886
|
+
catch (Exception ex)
|
|
887
|
+
{
|
|
888
|
+
_rendererFault = _rendererStage + ": " + ex.Message;
|
|
889
|
+
Program.AuditRecord("GLOW-RENDER-ERROR", _rendererFault);
|
|
890
|
+
}
|
|
891
|
+
finally
|
|
892
|
+
{
|
|
893
|
+
_rendererStage = "stopped";
|
|
894
|
+
lock (typeof(Glow)) { _thread = null; }
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
static void PumpOverlayMessages(IntPtr hwnd)
|
|
899
|
+
{
|
|
900
|
+
_rendererStage = "message-pump";
|
|
901
|
+
const uint WM_PAINT = 0x000F;
|
|
902
|
+
Native.NativeMsg msg;
|
|
903
|
+
for (int i = 0; i < 32 && PeekMessage(out msg, hwnd, 0, 0, PM_REMOVE); i++)
|
|
904
|
+
{
|
|
905
|
+
// UpdateLayeredWindow provides all pixels. WM_PAINT stays pending
|
|
906
|
+
// until its update region is validated, even with PM_REMOVE.
|
|
907
|
+
if (msg.message == WM_PAINT) ValidateRect(msg.hwnd, IntPtr.Zero);
|
|
908
|
+
else Native.DispatchMessage(ref msg);
|
|
909
|
+
}
|
|
711
910
|
}
|
|
712
911
|
|
|
713
912
|
static void Run()
|
|
714
913
|
{
|
|
914
|
+
while (RenderOverlay()) { } // Rebuild at most once per unpresented preparation.
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
static bool RenderOverlay()
|
|
918
|
+
{
|
|
919
|
+
_rendererFault = null;
|
|
920
|
+
_rendererStage = "display-metrics";
|
|
715
921
|
int vx = GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
716
922
|
int vy = GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
717
923
|
int vw = GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
718
924
|
int vh = GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
719
|
-
if (vw <= 0 || vh <= 0)
|
|
925
|
+
if (vw <= 0 || vh <= 0) throw new InvalidOperationException("empty virtual desktop");
|
|
720
926
|
|
|
927
|
+
_rendererStage = "create-window";
|
|
721
928
|
IntPtr hwnd = CreateWindowEx(
|
|
722
929
|
WS_EX_LAYERED | WS_EX_TRANSPARENT | WS_EX_TOOLWINDOW | WS_EX_NOACTIVATE | WS_EX_TOPMOST,
|
|
723
930
|
"STATIC", _label, WS_POPUP, vx, vy, vw, vh, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
|
|
724
|
-
if (hwnd == IntPtr.Zero)
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
931
|
+
if (hwnd == IntPtr.Zero) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
932
|
+
IntPtr screenDc = IntPtr.Zero, memDc = IntPtr.Zero, dib = IntPtr.Zero, oldBmp = IntPtr.Zero;
|
|
933
|
+
try
|
|
934
|
+
{
|
|
935
|
+
TryExcludeOverlayFromCapture(hwnd);
|
|
936
|
+
_hwnd = hwnd;
|
|
937
|
+
_rendererStage = "create-surface";
|
|
938
|
+
screenDc = GetDC(IntPtr.Zero);
|
|
939
|
+
if (screenDc == IntPtr.Zero) throw new InvalidOperationException("GetDC failed");
|
|
940
|
+
memDc = CreateCompatibleDC(screenDc);
|
|
941
|
+
if (memDc == IntPtr.Zero) throw new InvalidOperationException("CreateCompatibleDC failed");
|
|
942
|
+
Native.BITMAPINFO bmi = new Native.BITMAPINFO();
|
|
943
|
+
bmi.bmiHeader.biSize = (uint)Marshal.SizeOf(typeof(Native.BITMAPINFOHEADER));
|
|
944
|
+
bmi.bmiHeader.biWidth = vw;
|
|
945
|
+
bmi.bmiHeader.biHeight = -vh; // top-down
|
|
946
|
+
bmi.bmiHeader.biPlanes = 1;
|
|
947
|
+
bmi.bmiHeader.biBitCount = 32;
|
|
948
|
+
IntPtr bits;
|
|
949
|
+
dib = CreateDIBSection(screenDc, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
950
|
+
if (dib == IntPtr.Zero) throw new InvalidOperationException("CreateDIBSection failed");
|
|
951
|
+
oldBmp = SelectObject(memDc, dib);
|
|
952
|
+
if (oldBmp == IntPtr.Zero || oldBmp == new IntPtr(-1)) throw new InvalidOperationException("SelectObject failed");
|
|
953
|
+
_rendererStage = "rendering";
|
|
954
|
+
return RenderFrames(hwnd, screenDc, memDc, bits, vx, vy, vw, vh);
|
|
955
|
+
}
|
|
956
|
+
finally
|
|
957
|
+
{
|
|
958
|
+
AgentPointer.Dispose();
|
|
959
|
+
_hwnd = IntPtr.Zero; _shownTitle = null;
|
|
960
|
+
if (oldBmp != IntPtr.Zero && oldBmp != new IntPtr(-1)) SelectObject(memDc, oldBmp);
|
|
961
|
+
if (dib != IntPtr.Zero) DeleteObject(dib);
|
|
962
|
+
if (memDc != IntPtr.Zero) DeleteDC(memDc);
|
|
963
|
+
if (screenDc != IntPtr.Zero) ReleaseDC(IntPtr.Zero, screenDc);
|
|
964
|
+
DestroyWindow(hwnd);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
741
967
|
|
|
968
|
+
static bool RenderFrames(IntPtr hwnd, IntPtr screenDc, IntPtr memDc, IntPtr bits, int vx, int vy, int vw, int vh)
|
|
969
|
+
{
|
|
742
970
|
POINTL dst = new POINTL(); dst.x = vx; dst.y = vy;
|
|
743
971
|
POINTL src = new POINTL(); src.x = 0; src.y = 0;
|
|
744
972
|
SIZEL size = new SIZEL(); size.cx = vw; size.cy = vh;
|
|
@@ -746,14 +974,18 @@ namespace DshComputerUse.Worker
|
|
|
746
974
|
// ---- state machine with dwell hysteresis (no strobing) ----
|
|
747
975
|
int curBitmap = 0; // 0 = none, 1 = blue border, 2 = yellow flash
|
|
748
976
|
int paintedFlash = 0;
|
|
977
|
+
int lastPreparationPaint = 0;
|
|
749
978
|
int alpha = 0;
|
|
750
979
|
bool visible = false;
|
|
751
980
|
int committed = 0;
|
|
752
981
|
while (true)
|
|
753
982
|
{
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
983
|
+
_lastLoopTick = Environment.TickCount;
|
|
984
|
+
_rendererStage = "rendering";
|
|
985
|
+
Interlocked.Increment(ref _frames);
|
|
986
|
+
// Solid blue is actual input; breathing blue retains the scene between
|
|
987
|
+
// steps. Cyan is reserved for an explicitly free desktop.
|
|
988
|
+
bool acting = _enabled && _sceneVisible && Acting;
|
|
757
989
|
bool thinking = ThinkingActive;
|
|
758
990
|
bool flashOn = _flashStart != 0 && unchecked(Environment.TickCount - _flashStart) < _flashMs;
|
|
759
991
|
|
|
@@ -761,12 +993,14 @@ namespace DshComputerUse.Worker
|
|
|
761
993
|
|
|
762
994
|
// ONE derivation (Glow.DeriveWant): the exit flag outranks ask/stop/flash/committed,
|
|
763
995
|
// so no later state change on another thread can paint a light on an ended session.
|
|
996
|
+
int preparationSerial = Volatile.Read(ref _preparationSerial);
|
|
764
997
|
int want = DeriveWant(flashOn, committed);
|
|
765
998
|
_want = want; // publish it: the state the light is showing, as a fact
|
|
999
|
+
AgentPointer.Update(want == 1 || want == 8, want == 8);
|
|
766
1000
|
|
|
767
1001
|
// Keep the overlay's WINDOW TITLE in sync with the state, from the render loop.
|
|
768
|
-
//
|
|
769
|
-
//
|
|
1002
|
+
// When the title was only written by Stopped(), a process that INHERITED a stop
|
|
1003
|
+
// before this thread created the window (SyncFromFile at startup)
|
|
770
1004
|
// showed a red border with the title "AI 正在操控此电脑". The title is the text
|
|
771
1005
|
// channel the agent and the human both read, so a lying title is worse than none.
|
|
772
1006
|
// It is derived from the same flags (Glow.DeriveTitle) and a hidden window still has
|
|
@@ -782,16 +1016,17 @@ namespace DshComputerUse.Worker
|
|
|
782
1016
|
if (_killNow)
|
|
783
1017
|
{
|
|
784
1018
|
_killNow = false;
|
|
1019
|
+
AgentPointer.Hide();
|
|
785
1020
|
alpha = 0;
|
|
786
1021
|
if (visible) { ShowWindow(hwnd, SW_HIDE); visible = false; }
|
|
787
1022
|
curBitmap = 0; committed = 0; paintedFlash = 0;
|
|
788
|
-
|
|
1023
|
+
PumpOverlayMessages(hwnd);
|
|
789
1024
|
Thread.Sleep(12);
|
|
790
1025
|
continue;
|
|
791
1026
|
}
|
|
792
1027
|
// want: 0 idle | 1 acting (deep blue) | 3 thinking (pale blue) | 2 flash (yellow)
|
|
793
1028
|
// 4 emergency-stopped (RED, steady — the session is dead until re-armed)
|
|
794
|
-
int needBitmap = want == 7 ? 6 : want == 6 ? 5 : want == 2 ? 2 : (want == 0 ? curBitmap : (want == 1 ? 3 : (want == 4 || want == 5 ? 4 : 1)));
|
|
1029
|
+
int needBitmap = want == 7 ? 6 : want == 6 ? 5 : want == 2 ? 2 : (want == 0 ? curBitmap : ((want == 1 || want == 8) ? 3 : (want == 4 || want == 5 ? 4 : 1)));
|
|
795
1030
|
// A REPAINT MUST REACH THE SCREEN EVEN WHEN ALPHA DOES NOT MOVE.
|
|
796
1031
|
//
|
|
797
1032
|
// The present below used to be gated on `diff != 0` alone, while the flash snaps
|
|
@@ -802,7 +1037,14 @@ namespace DshComputerUse.Worker
|
|
|
802
1037
|
// RED frame when the border happened to be at full alpha already — whether the
|
|
803
1038
|
// brake was visible was luck. Present when EITHER the bitmap or the alpha changed.
|
|
804
1039
|
bool repaint = false;
|
|
805
|
-
|
|
1040
|
+
int preparationElapsed = -1;
|
|
1041
|
+
if (want == 7 && (curBitmap != 6 || !PreparationVisible || unchecked(Environment.TickCount - lastPreparationPaint) >= 50))
|
|
1042
|
+
{
|
|
1043
|
+
lastPreparationPaint = Environment.TickCount;
|
|
1044
|
+
preparationElapsed = PreparationVisible ? unchecked(lastPreparationPaint - _preparationStart) : 0;
|
|
1045
|
+
PaintPreparation(bits, vw, vh, preparationElapsed);
|
|
1046
|
+
curBitmap = 6; repaint = true;
|
|
1047
|
+
}
|
|
806
1048
|
else if (needBitmap == 2 && paintedFlash != _flashStart) { PaintFlash(bits, vw, vh); paintedFlash = _flashStart; curBitmap = 2; alpha = 204; repaint = true; /* snap ON at 80%: a flash is an event, not a fade */ }
|
|
807
1049
|
else if (needBitmap == 5 && curBitmap != 5) { PaintHumanYield(bits, vw, vh); curBitmap = 5; repaint = true; }
|
|
808
1050
|
else if (needBitmap == 4 && curBitmap != 4) { PaintStop(bits, vw, vh); curBitmap = 4; repaint = true; }
|
|
@@ -824,6 +1066,7 @@ namespace DshComputerUse.Worker
|
|
|
824
1066
|
flashFrame = true;
|
|
825
1067
|
}
|
|
826
1068
|
else if (want == 1) { target = FULL; breathFrame = true; }
|
|
1069
|
+
else if (want == 8) { target = HoldingAlpha(Environment.TickCount); breathFrame = true; }
|
|
827
1070
|
else if (want == 7) { target = 255; breathFrame = true; }
|
|
828
1071
|
else if (want == 4) target = FULL; // steady red, no breathing
|
|
829
1072
|
else if (want == 6)
|
|
@@ -834,22 +1077,23 @@ namespace DshComputerUse.Worker
|
|
|
834
1077
|
else if (want == 5)
|
|
835
1078
|
{
|
|
836
1079
|
// ASK: red like a stop, but at TWICE the cyan breath rate (1250 ms vs 2500 ms)
|
|
837
|
-
// and
|
|
1080
|
+
// and a full swing — so the pulse is unmistakable at a glance. Human spec
|
|
1081
|
+
// 2026-11-04, raised the same day after review: the first floor (25) still
|
|
1082
|
+
// read as too dim at the trough; the whole band moved up to 50..140.
|
|
838
1083
|
double askPhase = (Environment.TickCount % 1250) / 1250.0;
|
|
839
|
-
target =
|
|
1084
|
+
target = 95 + (int)(Math.Sin(askPhase * 2 * Math.PI) * 45);
|
|
840
1085
|
breathFrame = true;
|
|
841
1086
|
}
|
|
842
1087
|
else
|
|
843
1088
|
{
|
|
844
1089
|
// Faint blue "thinking", breathing so a still screen still shows life.
|
|
845
1090
|
// 2.5 s cycle = 20% faster than the original 3.0 s.
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
// point: the ring INHALES and goes dark, then comes back.
|
|
1091
|
+
// Human spec 2026-11-04, raised the same day after review: the first floor
|
|
1092
|
+
// (25) still read as "going dark" at the trough; the whole band moved up to
|
|
1093
|
+
// 50..140 — the halo stays clearly visible at its dimmest while the breathing
|
|
1094
|
+
// contrast (swing 90) is preserved.
|
|
851
1095
|
double phase = (Environment.TickCount % 2500) / 2500.0;
|
|
852
|
-
target =
|
|
1096
|
+
target = 95 + (int)(Math.Sin(phase * 2 * Math.PI) * 45);
|
|
853
1097
|
breathFrame = true;
|
|
854
1098
|
}
|
|
855
1099
|
|
|
@@ -886,37 +1130,93 @@ namespace DshComputerUse.Worker
|
|
|
886
1130
|
SetWindowPos(hwnd, new IntPtr(-1), vx, vy, vw, vh, 0x0004 | 0x0010 | 0x0040);
|
|
887
1131
|
visible = true;
|
|
888
1132
|
}
|
|
889
|
-
|
|
1133
|
+
_rendererStage = "presenting";
|
|
1134
|
+
Volatile.Write(ref _lastAttemptWant, want);
|
|
1135
|
+
Volatile.Write(ref _lastAttemptSerial, preparationSerial);
|
|
1136
|
+
if (want == 7) Volatile.Write(ref _lastAttemptedPreparationSerial, preparationSerial);
|
|
1137
|
+
Interlocked.Increment(ref _presentAttempts);
|
|
1138
|
+
bool presented = UpdateLayeredWindow(hwnd, screenDc, ref dst, ref size, memDc, ref src, 0, ref bf, ULW_ALPHA);
|
|
1139
|
+
int nativeError = presented ? 0 : Marshal.GetLastWin32Error();
|
|
1140
|
+
if (presented)
|
|
1141
|
+
{
|
|
1142
|
+
_lastPresentTick = Environment.TickCount;
|
|
1143
|
+
Interlocked.Increment(ref _presentSuccesses);
|
|
1144
|
+
}
|
|
1145
|
+
else
|
|
1146
|
+
{
|
|
1147
|
+
bool firstFailure = Volatile.Read(ref _lastFailedSerial) != preparationSerial;
|
|
1148
|
+
_lastFailedError = nativeError;
|
|
1149
|
+
Volatile.Write(ref _lastFailedWant, want);
|
|
1150
|
+
Volatile.Write(ref _lastFailedSerial, preparationSerial);
|
|
1151
|
+
if (firstFailure) Program.AuditRecord("GLOW-PRESENT-FAILED", "serial=" + preparationSerial + "; want=" + want + "; win32=" + nativeError);
|
|
1152
|
+
if (want == 7 && !PreparationVisible && Volatile.Read(ref _rebuiltSerial) != preparationSerial)
|
|
1153
|
+
{
|
|
1154
|
+
Volatile.Write(ref _rebuiltSerial, preparationSerial);
|
|
1155
|
+
return true;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
if (presented && want == 7 && !PreparationVisible)
|
|
890
1159
|
{
|
|
891
|
-
|
|
892
|
-
Volatile.Write(ref _presentedPreparation, Volatile.Read(ref _preparationSerial));
|
|
1160
|
+
PreparationPresented(preparationSerial);
|
|
893
1161
|
}
|
|
1162
|
+
if (presented && want == 7 && preparationElapsed >= PreparationMs)
|
|
1163
|
+
Volatile.Write(ref _completedPreparation, preparationSerial);
|
|
894
1164
|
}
|
|
895
1165
|
else if (visible) { ShowWindow(hwnd, SW_HIDE); visible = false; }
|
|
896
1166
|
}
|
|
897
|
-
|
|
898
|
-
while (PeekMessage(out m, hwnd, 0, 0, PM_REMOVE)) { }
|
|
1167
|
+
PumpOverlayMessages(hwnd);
|
|
899
1168
|
Thread.Sleep(16);
|
|
900
1169
|
}
|
|
901
1170
|
}
|
|
902
1171
|
|
|
903
1172
|
// Thin, square screen-edge rails leave the application itself completely clear.
|
|
904
1173
|
// Each pixel belongs to exactly one inset ring: corners meet without gaps or
|
|
905
|
-
// overlapping hot spots,
|
|
1174
|
+
// overlapping hot spots. Human request 2026-11-04, doubled the same day after
|
|
1175
|
+
// review: the bloom must span 1.6 cm of physical screen for every colour, not
|
|
1176
|
+
// the old fixed eight pixels (which read as a hairline on a 2560x1600 panel at
|
|
1177
|
+
// 175%), so the ring count below is derived from the overlay window's own DPI:
|
|
1178
|
+
// ~60 rings at 96 dpi, ~106 at 168.
|
|
1179
|
+
static int EdgeRailThickness()
|
|
1180
|
+
{
|
|
1181
|
+
// Offscreen painting (visual fixtures) has no window; GetDpiForWindow then
|
|
1182
|
+
// fails and the 96-dpi fallback keeps the fixture geometry deterministic.
|
|
1183
|
+
uint dpi = 96;
|
|
1184
|
+
IntPtr window = _hwnd;
|
|
1185
|
+
if (window != IntPtr.Zero)
|
|
1186
|
+
{
|
|
1187
|
+
uint live = Native.GetDpiForWindow(window);
|
|
1188
|
+
if (live > 0) dpi = live;
|
|
1189
|
+
}
|
|
1190
|
+
int rings = (int)Math.Round(1.6 / 2.54 * dpi); // 1.6 cm in this panel's pixels
|
|
1191
|
+
return Math.Max(8, Math.Min(rings, 192));
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
// The fixed eight-ring ramp was {244,216,94,53,30,16,8,3}: two bright core
|
|
1195
|
+
// rings, then a falloff to near zero. Same profile here, stretched over the
|
|
1196
|
+
// DPI-derived width: ring 0-1 keep their core alpha, rings 2..n-1 decay
|
|
1197
|
+
// geometrically from 94 down to ~3 at the innermost ring.
|
|
1198
|
+
static int EdgeRailAlpha(int ring, int rings)
|
|
1199
|
+
{
|
|
1200
|
+
if (ring == 0) return 244;
|
|
1201
|
+
if (ring == 1) return 216;
|
|
1202
|
+
return (int)Math.Round(94 * Math.Pow(3.0 / 94.0, (double)(ring - 2) / (rings - 3)));
|
|
1203
|
+
}
|
|
1204
|
+
|
|
906
1205
|
static void PaintEdgeRail(System.Drawing.Graphics g, Rectangle bounds, Color tint)
|
|
907
1206
|
{
|
|
908
|
-
int
|
|
1207
|
+
int rings = EdgeRailThickness();
|
|
909
1208
|
var smoothing = g.SmoothingMode;
|
|
910
1209
|
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.None;
|
|
911
1210
|
try
|
|
912
1211
|
{
|
|
913
|
-
for (int i = 0; i <
|
|
1212
|
+
for (int i = 0; i < rings; i++)
|
|
914
1213
|
{
|
|
1214
|
+
int alpha = EdgeRailAlpha(i, rings);
|
|
915
1215
|
int x = bounds.Left + i, y = bounds.Top + i;
|
|
916
1216
|
int width = bounds.Width - 2 * i, height = bounds.Height - 2 * i;
|
|
917
1217
|
if (width <= 0 || height <= 0) break;
|
|
918
|
-
Color start = Color.FromArgb(alpha
|
|
919
|
-
Color end = Color.FromArgb(alpha
|
|
1218
|
+
Color start = Color.FromArgb(alpha, Math.Min(255, tint.R + 18), Math.Min(255, tint.G + 14), Math.Min(255, tint.B + 8));
|
|
1219
|
+
Color end = Color.FromArgb(alpha, tint.R, tint.G, tint.B);
|
|
920
1220
|
using (var brush = new System.Drawing.Drawing2D.LinearGradientBrush(
|
|
921
1221
|
new RectangleF(bounds.X, bounds.Y, Math.Max(1, bounds.Width), Math.Max(1, bounds.Height)), start, end, 32f))
|
|
922
1222
|
{
|
|
@@ -959,6 +1259,9 @@ namespace DshComputerUse.Worker
|
|
|
959
1259
|
catch { /* display preference only; never changes control or capture behavior */ }
|
|
960
1260
|
}
|
|
961
1261
|
|
|
1262
|
+
public static int HoldingAlpha(int tick)
|
|
1263
|
+
{ return 120 + (int)Math.Round(42 * (1 - Math.Cos((unchecked((uint)tick) % 2500) * 2 * Math.PI / 2500.0))); }
|
|
1264
|
+
|
|
962
1265
|
static void PaintBlue(IntPtr bits, int w, int h, bool intense)
|
|
963
1266
|
{
|
|
964
1267
|
WithGraphics(bits, w, h, delegate(System.Drawing.Graphics g) {
|
|
@@ -968,7 +1271,7 @@ namespace DshComputerUse.Worker
|
|
|
968
1271
|
}
|
|
969
1272
|
|
|
970
1273
|
public static int HumanYieldAlpha(int tick)
|
|
971
|
-
{ return
|
|
1274
|
+
{ return 90 + (int)Math.Round(35 * (1 - Math.Cos((unchecked((uint)tick) % 2800) * 2 * Math.PI / 2800.0))); }
|
|
972
1275
|
static void PaintHumanYield(IntPtr bits, int w, int h)
|
|
973
1276
|
{
|
|
974
1277
|
WithGraphics(bits, w, h, delegate(System.Drawing.Graphics g) {
|
|
@@ -1252,7 +1555,9 @@ namespace DshComputerUse.Worker
|
|
|
1252
1555
|
_engaged = false;
|
|
1253
1556
|
_stopped = false;
|
|
1254
1557
|
CycleLit = false; // R2: the cycle is dead and must not be revived by anyone
|
|
1558
|
+
_controlsReleased = true;
|
|
1255
1559
|
Disarm(); // ... and its monitors die WITH it, not at the next turn
|
|
1560
|
+
SceneControl.Reset();
|
|
1256
1561
|
try { Glow.Kill(); } catch { }
|
|
1257
1562
|
try { Glow.Ask(false); } catch { }
|
|
1258
1563
|
try { Glow.Stopped(false); } catch { }
|
|
@@ -1283,7 +1588,8 @@ namespace DshComputerUse.Worker
|
|
|
1283
1588
|
// is what made the first signed op hide a brake the human had already set.
|
|
1284
1589
|
if (_engaged || _stopped)
|
|
1285
1590
|
{
|
|
1286
|
-
|
|
1591
|
+
SceneControl.Reset();
|
|
1592
|
+
try { Glow.Kill(); } catch { }
|
|
1287
1593
|
try { Glow.Stopped(true); } catch { }
|
|
1288
1594
|
// START the overlay too: `Thinking()` was the only thing that ever did, so a
|
|
1289
1595
|
// cycle whose FIRST call arrives with a brake already standing painted nothing at
|
|
@@ -1303,7 +1609,7 @@ namespace DshComputerUse.Worker
|
|
|
1303
1609
|
SyncExitedFromFile(); SyncFromFile();
|
|
1304
1610
|
lock (SyncLock)
|
|
1305
1611
|
{
|
|
1306
|
-
if (_engaged || _stopped || PauseRequested) return false;
|
|
1612
|
+
if (_engaged || _stopped || PauseRequested || ExitRequested) return false;
|
|
1307
1613
|
_controlsReleased = true;
|
|
1308
1614
|
CycleLit = false;
|
|
1309
1615
|
Disarm(); Glow.Kill();
|
|
@@ -1481,6 +1787,9 @@ namespace DshComputerUse.Worker
|
|
|
1481
1787
|
static IntPtr _hook = IntPtr.Zero;
|
|
1482
1788
|
static Native.LowLevelKeyboardProc _proc; // keep alive: the GC must not collect it
|
|
1483
1789
|
static IntPtr _mouseHook = IntPtr.Zero;
|
|
1790
|
+
static readonly uint CurrentPid = (uint)Process.GetCurrentProcess().Id;
|
|
1791
|
+
static readonly ManualResetEvent _hooksInitialized = new ManualResetEvent(false);
|
|
1792
|
+
static volatile bool _hooksReady;
|
|
1484
1793
|
static Native.LowLevelMouseProc _mouseProc; // same: the GC must not collect it
|
|
1485
1794
|
static Thread _thread;
|
|
1486
1795
|
|
|
@@ -1703,7 +2012,8 @@ namespace DshComputerUse.Worker
|
|
|
1703
2012
|
bool showRed = !_exited && CycleLit;
|
|
1704
2013
|
if (showRed)
|
|
1705
2014
|
{
|
|
1706
|
-
|
|
2015
|
+
SceneControl.Reset();
|
|
2016
|
+
try { Glow.Kill(); } catch { }
|
|
1707
2017
|
try { Glow.Stopped(true); } catch { }
|
|
1708
2018
|
try { Glow.Reveal(); } catch { }
|
|
1709
2019
|
}
|
|
@@ -1750,9 +2060,14 @@ namespace DshComputerUse.Worker
|
|
|
1750
2060
|
_proc = HookCallback;
|
|
1751
2061
|
IntPtr hmod = Native.GetModuleHandle(null);
|
|
1752
2062
|
_hook = Native.SetWindowsHookEx(Native.WH_KEYBOARD_LL, _proc, hmod, 0);
|
|
2063
|
+
int keyboardError = _hook == IntPtr.Zero ? Marshal.GetLastWin32Error() : 0;
|
|
1753
2064
|
// Same thread, same message loop: a second low-level hook for the wheel.
|
|
1754
2065
|
_mouseProc = MouseHookCallback;
|
|
1755
2066
|
_mouseHook = Native.SetWindowsHookEx(Native.WH_MOUSE_LL, _mouseProc, hmod, 0);
|
|
2067
|
+
int mouseError = _mouseHook == IntPtr.Zero ? Marshal.GetLastWin32Error() : 0;
|
|
2068
|
+
_hooksReady = _hook != IntPtr.Zero && _mouseHook != IntPtr.Zero;
|
|
2069
|
+
_hooksInitialized.Set();
|
|
2070
|
+
if (!_hooksReady) Program.AuditRecord("CONTROL-HOOK-ERROR", "keyboard=" + keyboardError + "; mouse=" + mouseError);
|
|
1756
2071
|
Native.NativeMsg msg;
|
|
1757
2072
|
while (Native.GetMessage(out msg, IntPtr.Zero, 0, 0) > 0)
|
|
1758
2073
|
{
|
|
@@ -1760,13 +2075,33 @@ namespace DshComputerUse.Worker
|
|
|
1760
2075
|
Native.DispatchMessage(ref msg);
|
|
1761
2076
|
}
|
|
1762
2077
|
}
|
|
1763
|
-
catch
|
|
2078
|
+
catch (Exception ex) { Program.AuditRecord("CONTROL-HOOK-ERROR", ex.GetType().Name); }
|
|
2079
|
+
finally { _hooksReady = false; _hooksInitialized.Set(); }
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
public static void RequireControlHooks()
|
|
2083
|
+
{
|
|
2084
|
+
if (!_hooksInitialized.WaitOne(1500) || !_hooksReady)
|
|
2085
|
+
{
|
|
2086
|
+
var error = new Exception("CONTROL_HOOK_UNAVAILABLE: physical input monitoring did not start or has stopped; no input was dispatched.");
|
|
2087
|
+
error.Data["code"] = "CONTROL_HOOK_UNAVAILABLE";
|
|
2088
|
+
error.Data["outcome"] = "not-dispatched";
|
|
2089
|
+
throw error;
|
|
2090
|
+
}
|
|
1764
2091
|
}
|
|
1765
2092
|
|
|
1766
2093
|
// Hook callbacks only record ownership and enqueue control work. Serial execution
|
|
1767
2094
|
// preserves Ctrl+Esc / resume / exit ordering without blocking the OS hook on I/O.
|
|
1768
|
-
static int _pendingPauses;
|
|
2095
|
+
static int _pendingPauses, _pendingExits;
|
|
1769
2096
|
public static bool PauseRequested { get { return Volatile.Read(ref _pendingPauses) > 0; } }
|
|
2097
|
+
public static bool ExitRequested { get { return Volatile.Read(ref _pendingExits) > 0; } }
|
|
2098
|
+
public static bool FinishOneShotControls()
|
|
2099
|
+
{
|
|
2100
|
+
if (!PauseRequested && !ExitRequested) return true;
|
|
2101
|
+
// A one-shot process must give the recognised chord a chance to persist
|
|
2102
|
+
// STOP/EXITED before Main returns and its background threads disappear.
|
|
2103
|
+
return SpinWait.SpinUntil(delegate { return !PauseRequested && !ExitRequested; }, 5000);
|
|
2104
|
+
}
|
|
1770
2105
|
static readonly ConcurrentQueue<Action> ControlQueue = new ConcurrentQueue<Action>();
|
|
1771
2106
|
static int _controlWorkQueued;
|
|
1772
2107
|
internal static void QueueControl(Action action)
|
|
@@ -1781,20 +2116,31 @@ namespace DshComputerUse.Worker
|
|
|
1781
2116
|
} while (!ControlQueue.IsEmpty && Interlocked.CompareExchange(ref _controlWorkQueued, 1, 0) == 0);
|
|
1782
2117
|
});
|
|
1783
2118
|
}
|
|
1784
|
-
static void QueueDesktopControl(Action action, Action
|
|
2119
|
+
static void QueueDesktopControl(Action action, Action finished = null)
|
|
1785
2120
|
{
|
|
1786
2121
|
QueueControl(delegate {
|
|
1787
2122
|
// Serialize with graceful release: a queued physical hotkey may not
|
|
1788
2123
|
// reopen a completed cycle or steer an exit into its continuing chat.
|
|
1789
|
-
|
|
2124
|
+
try
|
|
1790
2125
|
{
|
|
1791
|
-
|
|
1792
|
-
{
|
|
1793
|
-
|
|
1794
|
-
|
|
2126
|
+
lock (SyncLock)
|
|
2127
|
+
{
|
|
2128
|
+
if (!ControlListenersEnabled || !Program.TryClaimDesktop("physical control"))
|
|
2129
|
+
{ Program.AuditRecord("CONTROL-SKIPPED", ControlListenersEnabled ? "desktop owned by another worker" : "cycle released"); return; }
|
|
2130
|
+
try { action(); }
|
|
2131
|
+
finally { if (!CycleLit) Program.ReleaseDesktopOwnership(); }
|
|
2132
|
+
}
|
|
1795
2133
|
}
|
|
2134
|
+
catch (Exception ex) { Program.AuditRecord("CONTROL-ERROR", ex.GetType().Name); }
|
|
2135
|
+
finally { if (finished != null) finished(); }
|
|
1796
2136
|
});
|
|
1797
2137
|
}
|
|
2138
|
+
|
|
2139
|
+
static void QueueExit(Action exit)
|
|
2140
|
+
{
|
|
2141
|
+
Interlocked.Increment(ref _pendingExits);
|
|
2142
|
+
QueueDesktopControl(exit, delegate { Interlocked.Decrement(ref _pendingExits); });
|
|
2143
|
+
}
|
|
1798
2144
|
public static string ControlHotkey(uint vk, bool down, bool human, bool ctrl, bool alt, bool shift, bool win, bool live)
|
|
1799
2145
|
{
|
|
1800
2146
|
if (!down || !human) return "";
|
|
@@ -1818,16 +2164,22 @@ namespace DshComputerUse.Worker
|
|
|
1818
2164
|
if (m.dwExtraInfo != OwnMagic) Interlocked.Increment(ref _externalInput);
|
|
1819
2165
|
bool physical = PhysicalEvent(m.dwExtraInfo, m.flags, 3);
|
|
1820
2166
|
int message = wParam.ToInt32();
|
|
2167
|
+
if (!physical) {
|
|
2168
|
+
Interlocked.Exchange(ref _lastInjectedInputAt, HumanClock.ElapsedMilliseconds);
|
|
2169
|
+
int injectedKey = message == 0x201 || message == 0x202 ? 1 : message == 0x204 || message == 0x205 ? 2 :
|
|
2170
|
+
message == 0x207 || message == 0x208 ? 4 : message == 0x20B || message == 0x20C ? (((m.mouseData >> 16) & 0xFFFF) == 1 ? 5 : 6) : 0;
|
|
2171
|
+
if (injectedKey != 0) Handoff.NoteInjected(injectedKey);
|
|
2172
|
+
}
|
|
1821
2173
|
// Significant idle motion invalidates observations, but does not yield.
|
|
1822
2174
|
bool movement = message == 0x200 && PointerMotion.Record(m.pt.X, m.pt.Y, physical,
|
|
1823
2175
|
true, Handoff.Active, HumanClock.ElapsedMilliseconds);
|
|
1824
2176
|
if (physical)
|
|
1825
2177
|
{
|
|
1826
2178
|
int key = 0;
|
|
1827
|
-
bool down = false, up = false, activity = true;
|
|
2179
|
+
bool down = false, up = false, activity = true, deliberate = false;
|
|
1828
2180
|
switch (message)
|
|
1829
2181
|
{
|
|
1830
|
-
case 0x200: activity = movement; break;
|
|
2182
|
+
case 0x200: activity = movement; deliberate = movement; break;
|
|
1831
2183
|
case 0x201: key = 1; down = true; break;
|
|
1832
2184
|
case 0x202: key = 1; up = true; break;
|
|
1833
2185
|
case 0x204: key = 2; down = true; break;
|
|
@@ -1839,7 +2191,7 @@ namespace DshComputerUse.Worker
|
|
|
1839
2191
|
case 0x20A: case 0x20E: activity = unchecked((short)(m.mouseData >> 16)) != 0; break;
|
|
1840
2192
|
default: activity = false; break;
|
|
1841
2193
|
}
|
|
1842
|
-
if (activity) RecordHumanInput(key, down, up);
|
|
2194
|
+
if (activity) RecordHumanInput(key, down, up, deliberate);
|
|
1843
2195
|
}
|
|
1844
2196
|
}
|
|
1845
2197
|
}
|
|
@@ -1859,6 +2211,10 @@ namespace DshComputerUse.Worker
|
|
|
1859
2211
|
bool down = m == Native.WM_KEYDOWN_ || m == Native.WM_SYSKEYDOWN_;
|
|
1860
2212
|
bool up = m == Native.WM_KEYUP_ || m == Native.WM_SYSKEYUP_;
|
|
1861
2213
|
bool human = ClassifyAndTrackKey(k.vkCode, down, up, k.dwExtraInfo, k.flags);
|
|
2214
|
+
if (!human && (down || up)) {
|
|
2215
|
+
Interlocked.Exchange(ref _lastInjectedInputAt, HumanClock.ElapsedMilliseconds);
|
|
2216
|
+
Handoff.NoteInjected((int)k.vkCode);
|
|
2217
|
+
}
|
|
1862
2218
|
if (human && (down || up))
|
|
1863
2219
|
{
|
|
1864
2220
|
RecordHumanInput((int)k.vkCode, down, up);
|
|
@@ -1871,20 +2227,18 @@ namespace DshComputerUse.Worker
|
|
|
1871
2227
|
if (hotkey == "pause")
|
|
1872
2228
|
{
|
|
1873
2229
|
_suppressPauseEsc = true;
|
|
1874
|
-
Interlocked.Increment(ref _pendingPauses);
|
|
1875
2230
|
// The hook marks ownership immediately; do not perform SendInput or
|
|
1876
2231
|
// filesystem work on this OS hook thread.
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
}, delegate { Interlocked.Decrement(ref _pendingPauses); });
|
|
2232
|
+
var evidence = ChordEvidence(k, ctrl, alt);
|
|
2233
|
+
Interlocked.Increment(ref _pendingPauses);
|
|
2234
|
+
QueueDesktopControl(delegate { Engage("Ctrl+Esc chord detected", false, "hook", evidence()); },
|
|
2235
|
+
delegate { Interlocked.Decrement(ref _pendingPauses); });
|
|
1882
2236
|
return new IntPtr(1); // Ctrl+Esc must not also open the Windows Start menu.
|
|
1883
2237
|
}
|
|
1884
2238
|
if (hotkey == "exit")
|
|
1885
2239
|
{
|
|
1886
|
-
|
|
1887
|
-
|
|
2240
|
+
var evidence = ChordEvidence(k, ctrl, alt);
|
|
2241
|
+
QueueExit(delegate { Exit("Ctrl+Alt+Q chord detected", "hook", evidence()); });
|
|
1888
2242
|
}
|
|
1889
2243
|
else if (hotkey == "resume")
|
|
1890
2244
|
QueueDesktopControl(delegate { Clear(); });
|
|
@@ -1940,7 +2294,7 @@ namespace DshComputerUse.Worker
|
|
|
1940
2294
|
/// The chord as it was SEEN: the fields someone can re-check, including whose Ctrl/Alt the
|
|
1941
2295
|
/// hook recorded and which window was in front. Deliberately verbose — this line is the
|
|
1942
2296
|
/// difference between "the human pressed it" and "we cannot know who pressed it".
|
|
1943
|
-
static string ChordEvidence(Native.KBDLLHOOKSTRUCT k, bool ctrl, bool alt)
|
|
2297
|
+
static Func<string> ChordEvidence(Native.KBDLLHOOKSTRUCT k, bool ctrl, bool alt)
|
|
1944
2298
|
{
|
|
1945
2299
|
StringBuilder sb = new StringBuilder();
|
|
1946
2300
|
sb.Append("vk=0x").Append(k.vkCode.ToString("X2"));
|
|
@@ -1954,31 +2308,37 @@ namespace DshComputerUse.Worker
|
|
|
1954
2308
|
// later whether "the human pressed it" was ever measurable — and note that OURSELVES
|
|
1955
2309
|
// (extra=OUR_MAGIC) is a different, stronger answer than "some injector".
|
|
1956
2310
|
sb.Append(" injected=").Append((k.flags & Native.LLKHF_INJECTED) != 0 ? "1" : "0");
|
|
1957
|
-
|
|
2311
|
+
IntPtr fg = Native.GetForegroundWindow();
|
|
2312
|
+
uint fgPid;
|
|
2313
|
+
Native.GetWindowThreadProcessId(fg, out fgPid);
|
|
2314
|
+
sb.Append(" fgPid=").Append(fgPid);
|
|
2315
|
+
// Capture the foreground identity on the hook thread. Enrich it on the
|
|
2316
|
+
// control queue without sending a message to our own overlay window.
|
|
2317
|
+
return delegate
|
|
1958
2318
|
{
|
|
1959
|
-
|
|
1960
|
-
uint fgPid;
|
|
1961
|
-
Native.GetWindowThreadProcessId(fg, out fgPid);
|
|
1962
|
-
sb.Append(" fgPid=").Append(fgPid);
|
|
1963
|
-
try { sb.Append(" fg=").Append(Process.GetProcessById((int)fgPid).ProcessName); } catch { }
|
|
1964
|
-
StringBuilder t = new StringBuilder(256);
|
|
1965
|
-
if (Native.GetWindowText(fg, t, 256) > 0)
|
|
2319
|
+
try
|
|
1966
2320
|
{
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
2321
|
+
using (var proc = Process.GetProcessById((int)fgPid)) sb.Append(" fg=").Append(proc.ProcessName);
|
|
2322
|
+
uint currentPid;
|
|
2323
|
+
Native.GetWindowThreadProcessId(fg, out currentPid);
|
|
2324
|
+
StringBuilder t = new StringBuilder(256);
|
|
2325
|
+
if (currentPid == fgPid && fgPid != CurrentPid && Native.GetWindowText(fg, t, 256) > 0)
|
|
2326
|
+
{
|
|
2327
|
+
string title = t.ToString();
|
|
2328
|
+
if (title.Length > 60) title = title.Substring(0, 60);
|
|
2329
|
+
sb.Append(" fgTitle=\"").Append(title.Replace("\t", " ")).Append('"');
|
|
2330
|
+
}
|
|
1970
2331
|
}
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
2332
|
+
catch { }
|
|
2333
|
+
return sb.ToString();
|
|
2334
|
+
};
|
|
1974
2335
|
}
|
|
1975
2336
|
|
|
1976
2337
|
/// Polling fallback for the human's combos.
|
|
1977
2338
|
///
|
|
1978
2339
|
/// The low-level hook is the primary path, but it only reports what Windows hands it: a
|
|
1979
|
-
/// missed keydown would
|
|
1980
|
-
///
|
|
1981
|
-
/// missed when polled every 40 ms — so Q always exits and R always releases, hook or no hook.
|
|
2340
|
+
/// missed keydown would otherwise have no fallback. Poll the press bits as an additional
|
|
2341
|
+
/// witness for Q/R while the hook has confirmed human-held modifiers.
|
|
1982
2342
|
static void PollThread()
|
|
1983
2343
|
{
|
|
1984
2344
|
bool prevQ = false, prevR = false;
|
|
@@ -2020,11 +2380,9 @@ namespace DshComputerUse.Worker
|
|
|
2020
2380
|
{
|
|
2021
2381
|
Native.KBDLLHOOKSTRUCT kk = new Native.KBDLLHOOKSTRUCT();
|
|
2022
2382
|
kk.vkCode = Native.VK_Q;
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
ChordEvidence(kk, asyncCtrl, asyncAlt);
|
|
2027
|
-
QueueDesktopControl(delegate { Exit("Ctrl+Alt+Q chord detected", "poll", evidence); });
|
|
2383
|
+
var evidence = ChordEvidence(kk, asyncCtrl, asyncAlt);
|
|
2384
|
+
QueueExit(delegate { Exit("Ctrl+Alt+Q chord detected", "poll",
|
|
2385
|
+
"q-latch + hook-tracked modifiers + measured async key state; " + evidence()); });
|
|
2028
2386
|
}
|
|
2029
2387
|
}
|
|
2030
2388
|
prevQ = q; prevR = r;
|
|
@@ -2046,6 +2404,7 @@ namespace DshComputerUse.Worker
|
|
|
2046
2404
|
public static bool Armed { get { return _armed; } }
|
|
2047
2405
|
public static bool Exited { get { return _exited; } }
|
|
2048
2406
|
static readonly Stopwatch HumanClock = Stopwatch.StartNew();
|
|
2407
|
+
static long _lastInjectedInputAt;
|
|
2049
2408
|
static readonly HumanInputState Handoff = new HumanInputState();
|
|
2050
2409
|
static readonly object HandoffPublishLock = new object();
|
|
2051
2410
|
static string _publishedHandoff = "idle";
|
|
@@ -2059,36 +2418,78 @@ namespace DshComputerUse.Worker
|
|
|
2059
2418
|
{
|
|
2060
2419
|
var state = (Dictionary<string, object>)Handoff.Snapshot(HumanClock.ElapsedMilliseconds);
|
|
2061
2420
|
state["inputEpoch"] = InputEpoch; state["mouseTolerancePx"] = PointerHandoffTolerance.DistancePx;
|
|
2421
|
+
state["workerGeneration"] = Program.WorkerGeneration;
|
|
2062
2422
|
return state;
|
|
2063
2423
|
}
|
|
2064
2424
|
static void ResetMonitorRun() { /* Manual recovery must not erase physical input history. */ }
|
|
2425
|
+
// ---------- secure-desktop (UAC / lock screen) dispatch gate, human request 2026-11-04 ----------
|
|
2426
|
+
static readonly SecureDesktopWatch SecureWatch = new SecureDesktopWatch();
|
|
2427
|
+
public static bool SecureDesktopActive { get { return SecureWatch.Active; } }
|
|
2428
|
+
static bool InputDesktopReachable()
|
|
2429
|
+
{
|
|
2430
|
+
IntPtr desktop = Native.OpenInputDesktop(0, false, 0x0001); // DESKTOP_READOBJECT
|
|
2431
|
+
if (desktop != IntPtr.Zero) { Native.CloseDesktop(desktop); return true; }
|
|
2432
|
+
return false;
|
|
2433
|
+
}
|
|
2434
|
+
static bool OnNormalInputDesktop()
|
|
2435
|
+
{
|
|
2436
|
+
// Async state may fail when the caller lacks desktop hook access.
|
|
2437
|
+
IntPtr desktop = Native.OpenInputDesktop(0, false, 0x0001 | 0x0008);
|
|
2438
|
+
if (desktop == IntPtr.Zero) return false;
|
|
2439
|
+
try {
|
|
2440
|
+
var current = new StringBuilder(256); var input = new StringBuilder(256); int needed;
|
|
2441
|
+
return Native.GetUserObjectInformation(Native.GetThreadDesktop(Native.GetCurrentThreadId()), 2, current, 512, out needed) &&
|
|
2442
|
+
Native.GetUserObjectInformation(desktop, 2, input, 512, out needed) &&
|
|
2443
|
+
String.Equals(current.ToString(), input.ToString(), StringComparison.OrdinalIgnoreCase);
|
|
2444
|
+
} finally { Native.CloseDesktop(desktop); }
|
|
2445
|
+
}
|
|
2446
|
+
/// Polled from the monitor thread. Audits and notifies TRANSITIONS only.
|
|
2447
|
+
static void PollSecureDesktop()
|
|
2448
|
+
{
|
|
2449
|
+
if (!SecureWatch.Observe(InputDesktopReachable())) return;
|
|
2450
|
+
bool active = SecureWatch.Active;
|
|
2451
|
+
Program.AuditRecord(active ? "SECURE-DESKTOP-ENGAGED" : "SECURE-DESKTOP-CLEARED",
|
|
2452
|
+
active
|
|
2453
|
+
? "input desktop unreachable (UAC prompt or lock screen); input dispatch paused until it returns"
|
|
2454
|
+
: "input desktop reachable again; dispatch may continue");
|
|
2455
|
+
Notify("secure", active ? "active" : "cleared");
|
|
2456
|
+
}
|
|
2457
|
+
/// Input ops refuse to dispatch while the secure desktop stands: no injection can reach
|
|
2458
|
+
/// it anyway, and a clean refusal plus a notified human beats a blind timeout mid-cycle.
|
|
2459
|
+
public static void RequireNormalDesktop()
|
|
2460
|
+
{
|
|
2461
|
+
if (!SecureWatch.Active) return;
|
|
2462
|
+
var error = new Exception("SECURE_DESKTOP_ACTIVE: the Windows secure desktop (a UAC prompt or the lock screen) is up; no input was dispatched. Ask the human to answer the prompt; dispatch resumes automatically once the normal desktop returns.");
|
|
2463
|
+
error.Data["code"] = "SECURE_DESKTOP_ACTIVE";
|
|
2464
|
+
error.Data["outcome"] = "not-dispatched";
|
|
2465
|
+
throw error;
|
|
2466
|
+
}
|
|
2065
2467
|
public static void Arm() { _armed = true; Handoff.Enable(CycleLit && !_exited, HumanClock.ElapsedMilliseconds); }
|
|
2066
|
-
public static void Disarm() { _armed = false; Handoff.Enable(false, HumanClock.ElapsedMilliseconds); PublishHandoff(); }
|
|
2468
|
+
public static void Disarm() { _armed = false; SceneControl.Reset(); Handoff.Enable(false, HumanClock.ElapsedMilliseconds); PublishHandoff(); }
|
|
2067
2469
|
public static void AgentActed() { _agentActAt = Environment.TickCount; }
|
|
2068
2470
|
public static void BeginActuation()
|
|
2069
2471
|
{
|
|
2070
2472
|
POINT point; if (Native.GetCursorPos(out point)) PointerMotion.Reset(point.X, point.Y);
|
|
2071
|
-
Glow.ActuationBegin();
|
|
2473
|
+
SceneControl.BeginActuation(); Glow.ActuationBegin();
|
|
2072
2474
|
Handoff.SetActing(true, HumanClock.ElapsedMilliseconds);
|
|
2073
2475
|
}
|
|
2074
|
-
public static void EndActuation()
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
Glow.ActuationEnd();
|
|
2078
|
-
}
|
|
2476
|
+
public static void EndActuation() { SceneControl.EndActuation(); Glow.ActuationEnd(); }
|
|
2477
|
+
public static void UpdateInputProtection()
|
|
2478
|
+
{ Handoff.SetActing(SceneControl.Acting || SceneControl.Holding, HumanClock.ElapsedMilliseconds); }
|
|
2079
2479
|
static bool IsBareModifier(uint vk)
|
|
2080
2480
|
{
|
|
2081
2481
|
return vk == Native.VK_SHIFT || vk == Native.VK_CONTROL || vk == Native.VK_MENU ||
|
|
2082
2482
|
vk == 0xA0 || vk == 0xA1 || vk == 0xA2 || vk == 0xA3 ||
|
|
2083
2483
|
vk == 0xA4 || vk == 0xA5 || vk == 0x5B || vk == 0x5C;
|
|
2084
2484
|
}
|
|
2085
|
-
public static void RecordHumanInput(int key, bool down, bool up)
|
|
2485
|
+
public static void RecordHumanInput(int key, bool down, bool up) { RecordHumanInput(key, down, up, false); }
|
|
2486
|
+
public static void RecordHumanInput(int key, bool down, bool up, bool deliberate)
|
|
2086
2487
|
{
|
|
2087
2488
|
Interlocked.Increment(ref _physicalInputEpoch);
|
|
2088
2489
|
long now = HumanClock.ElapsedMilliseconds;
|
|
2089
2490
|
bool was = Handoff.Active;
|
|
2090
2491
|
Handoff.Enable(_armed && CycleLit && !_exited, now);
|
|
2091
|
-
Handoff.Record(key, down, up, now);
|
|
2492
|
+
Handoff.Record(key, down, up, now, deliberate);
|
|
2092
2493
|
if (!was && Handoff.Active && Interlocked.CompareExchange(ref _handoffWorkQueued, 1, 0) == 0)
|
|
2093
2494
|
ThreadPool.QueueUserWorkItem(delegate {
|
|
2094
2495
|
try { Program.ReleaseOwnedInputs(); PublishHandoff(); }
|
|
@@ -2107,7 +2508,12 @@ namespace DshComputerUse.Worker
|
|
|
2107
2508
|
_publishedInputEpoch = epoch;
|
|
2108
2509
|
if (!ControlListenersEnabled && !CycleLit && phase == "idle") return;
|
|
2109
2510
|
if (phase != "idle") Glow.HumanYield();
|
|
2110
|
-
if (changed)
|
|
2511
|
+
if (changed) {
|
|
2512
|
+
var state = (Dictionary<string, object>)CooperationState();
|
|
2513
|
+
Program.AuditRecord("HUMAN-HANDOFF", "phase=" + phase + "; revision=" + state["revision"] +
|
|
2514
|
+
"; kind=" + state["lastKind"] + "; heldCount=" + state["heldCount"] +
|
|
2515
|
+
"; quietRemainingMs=" + state["quietRemainingMs"] + "; inputEpoch=" + state["inputEpoch"]);
|
|
2516
|
+
}
|
|
2111
2517
|
Notify("handoff", phase);
|
|
2112
2518
|
}
|
|
2113
2519
|
}
|
|
@@ -2115,16 +2521,40 @@ namespace DshComputerUse.Worker
|
|
|
2115
2521
|
{
|
|
2116
2522
|
if (!ControlListenersEnabled) return;
|
|
2117
2523
|
SyncExitedFromFile();
|
|
2524
|
+
PollSecureDesktop();
|
|
2118
2525
|
long now = HumanClock.ElapsedMilliseconds;
|
|
2119
2526
|
Handoff.Enable(_armed && CycleLit && !_exited, now);
|
|
2527
|
+
IntPtr sampleForeground = IntPtr.Zero;
|
|
2528
|
+
bool trusted = Handoff.HasHeld && CanReconcileHeld(_hooksReady, _thread != null && _thread.IsAlive, SecureDesktopActive,
|
|
2529
|
+
SceneControl.Acting, Glow.Injecting, now - Interlocked.Read(ref _lastInjectedInputAt), OnNormalInputDesktop()) &&
|
|
2530
|
+
Program.CanObserveAsyncKeys(out sampleForeground);
|
|
2531
|
+
bool swapped = Native.GetSystemMetrics(23) != 0; // SM_SWAPBUTTON
|
|
2532
|
+
int reconciled = Handoff.ReconcileReleased(delegate(int key) {
|
|
2533
|
+
bool down = HumanInputState.AsyncButtonMayBeDown(key, swapped,
|
|
2534
|
+
delegate(int physical) { return (Native.GetAsyncKeyState(physical) & 0x8000) != 0; });
|
|
2535
|
+
// A foreground/desktop switch during the sample makes zero unknown,
|
|
2536
|
+
// never evidence of release. The next tick checks the new token.
|
|
2537
|
+
return down || Native.GetForegroundWindow() != sampleForeground || !OnNormalInputDesktop();
|
|
2538
|
+
}, trusted, now);
|
|
2539
|
+
if (reconciled > 0) Program.AuditRecord("HUMAN-RELEASE-RECONCILED", "count=" + reconciled + "; source=three-normal-desktop-async-up-samples; quietMs=" + HumanInputState.IdleMs);
|
|
2120
2540
|
Handoff.Tick(now);
|
|
2121
2541
|
if (Handoff.Active) Program.ReleaseOwnedInputs();
|
|
2122
2542
|
PublishHandoff();
|
|
2123
2543
|
}
|
|
2544
|
+
static bool CanReconcileHeld(bool hooksReady, bool hookAlive, bool secure, bool acting, bool injecting, long sinceInjected, bool inputDesktop)
|
|
2545
|
+
{ return hooksReady && hookAlive && !secure && !acting && !injecting && sinceInjected >= 300 && inputDesktop; }
|
|
2124
2546
|
|
|
2125
2547
|
static RecoveryPermit _temporary;
|
|
2126
2548
|
static long _externalInput;
|
|
2127
2549
|
public static string TemporaryPauseId { get { lock (SyncLock) { return _temporary == null ? "" : _temporary.Id; } } }
|
|
2550
|
+
public static string EventPauseId(string kind)
|
|
2551
|
+
{
|
|
2552
|
+
// Handoff events are serialized under HandoffPublishLock. Lifecycle
|
|
2553
|
+
// transitions take SyncLock then publish, so their notification must
|
|
2554
|
+
// not take SyncLock in the reverse order. These two observation events
|
|
2555
|
+
// neither own nor consume temporary-pause credentials.
|
|
2556
|
+
return kind == "handoff" || kind == "secure" ? "" : TemporaryPauseId;
|
|
2557
|
+
}
|
|
2128
2558
|
public static void ObserveTemporary(string id)
|
|
2129
2559
|
{
|
|
2130
2560
|
lock (SyncLock) { if (_temporary != null && _temporary.Id == id) _temporary.Observed = true; }
|
|
@@ -2200,6 +2630,7 @@ namespace DshComputerUse.Worker
|
|
|
2200
2630
|
InputVersion = Interlocked.Read(ref _externalInput), InputTick = Native.LastInputTick() };
|
|
2201
2631
|
WriteStopFile(why + "\nsource=agent-temporary\npauseId=" + _temporary.Id);
|
|
2202
2632
|
}
|
|
2633
|
+
SceneControl.Reset();
|
|
2203
2634
|
try { Glow.Kill(); } catch { }
|
|
2204
2635
|
try { Glow.Stopped(true); } catch { } // red border + red badge: unmistakable
|
|
2205
2636
|
// THE BRAKE IS A STATE CHANGE, SO IT GETS A LINE (2026-09-14). This is the record the
|
|
@@ -2262,6 +2693,7 @@ namespace DshComputerUse.Worker
|
|
|
2262
2693
|
Program.AuditRecord("ENGAGE-ASK", "source=agent" +
|
|
2263
2694
|
"; askPid=" + Process.GetCurrentProcess().Id +
|
|
2264
2695
|
"; cycleLit=" + (CycleLit ? 1 : 0) + "; why=" + why);
|
|
2696
|
+
SceneControl.Reset();
|
|
2265
2697
|
try { Glow.Kill(); } catch { }
|
|
2266
2698
|
try { Glow.Ask(true); } catch { } // fast-breathing RED, not the steady stop
|
|
2267
2699
|
announce = true;
|
|
@@ -2444,7 +2876,7 @@ namespace DshComputerUse.Worker
|
|
|
2444
2876
|
static readonly HashSet<string> ReadOnlyOps = new HashSet<string> {
|
|
2445
2877
|
"selftest", "cursor", "panic", "resume", "recover", "echoargs", "ping", "screens",
|
|
2446
2878
|
"windows", "uia", "uiaFromPoint", "uiaFocused", "waitForIdle", "clipRead",
|
|
2447
|
-
"capture", "annotate", "waitStable", "waitChange", "focus", "frameSig", "probe",
|
|
2879
|
+
"capture", "ocr", "annotate", "waitStable", "waitChange", "focus", "frameSig", "probe",
|
|
2448
2880
|
// diagnostics that must be able to ANSWER while a brake is up: they send no input and
|
|
2449
2881
|
// light nothing, which is exactly when someone needs to inspect the release decision
|
|
2450
2882
|
"keyClass", "stuckPlan",
|
|
@@ -2532,6 +2964,7 @@ namespace DshComputerUse.Worker
|
|
|
2532
2964
|
try { Glow.Stopped(false); } catch { }
|
|
2533
2965
|
// Re-arming re-baselines the human monitor: the travel that tripped the brake must not
|
|
2534
2966
|
// still be sitting in the accumulator, or the very next poll would brake again.
|
|
2967
|
+
if (CycleLit && !_exited) Glow.Thinking(0);
|
|
2535
2968
|
_agentActAt = 0;
|
|
2536
2969
|
}
|
|
2537
2970
|
if (was) Notify("resume", "");
|
|
@@ -2693,6 +3126,51 @@ namespace DshComputerUse.Worker
|
|
|
2693
3126
|
}
|
|
2694
3127
|
}
|
|
2695
3128
|
|
|
3129
|
+
// Only complete window-management chords may change their own foreground receiver.
|
|
3130
|
+
// Every down still passes the ordinary receiver guard. After the final accepted down,
|
|
3131
|
+
// only this chord's owned releases may bypass that guard; human/pause/cancel still win.
|
|
3132
|
+
public static class WindowManagementChord
|
|
3133
|
+
{
|
|
3134
|
+
public static bool IsExact(string canonical)
|
|
3135
|
+
{
|
|
3136
|
+
switch (canonical)
|
|
3137
|
+
{
|
|
3138
|
+
case "alt+tab": case "alt+shift+tab": case "alt+esc": case "alt+shift+esc":
|
|
3139
|
+
case "win+d": case "win+tab": case "win+left": case "win+right": case "win+up": case "win+down":
|
|
3140
|
+
case "rwin+d": case "rwin+tab": case "rwin+left": case "rwin+right": case "rwin+up": case "rwin+down":
|
|
3141
|
+
return true;
|
|
3142
|
+
default: return false;
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
|
|
3146
|
+
public static void Run(IList<ushort> keys, Action<ushort, bool> send,
|
|
3147
|
+
Action<ushort> release, Func<ushort, bool> isDown, Action hold,
|
|
3148
|
+
Action checkContinuation, Action invalidateTarget)
|
|
3149
|
+
{
|
|
3150
|
+
int acceptedDowns = 0;
|
|
3151
|
+
try
|
|
3152
|
+
{
|
|
3153
|
+
KeyboardChord.Run(keys, delegate(ushort key, bool up) {
|
|
3154
|
+
if (up && acceptedDowns == keys.Count)
|
|
3155
|
+
{
|
|
3156
|
+
checkContinuation();
|
|
3157
|
+
release(key); // ledger-backed: never releases someone else's key
|
|
3158
|
+
}
|
|
3159
|
+
else
|
|
3160
|
+
{
|
|
3161
|
+
send(key, up);
|
|
3162
|
+
if (!up) acceptedDowns++;
|
|
3163
|
+
}
|
|
3164
|
+
}, release, isDown, hold);
|
|
3165
|
+
}
|
|
3166
|
+
finally
|
|
3167
|
+
{
|
|
3168
|
+
// Even a partial chord can open system UI. No stale sticky/refocus continuation.
|
|
3169
|
+
if (acceptedDowns > 0) invalidateTarget();
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3173
|
+
|
|
2696
3174
|
public sealed class ClipboardBytes
|
|
2697
3175
|
{
|
|
2698
3176
|
public uint Format;
|
|
@@ -2721,12 +3199,16 @@ namespace DshComputerUse.Worker
|
|
|
2721
3199
|
bool finished;
|
|
2722
3200
|
ClipboardLease(IClipboardAccess access, List<ClipboardBytes> previous, uint written)
|
|
2723
3201
|
{ this.access = access; this.previous = previous; this.written = written; }
|
|
3202
|
+
/// Returns null when the previous clipboard cannot be preserved (private formats, odd
|
|
3203
|
+
/// sizes). The caller then downgrades to a plain overwrite-paste with a warning rather
|
|
3204
|
+
/// than refusing — see Snapshot's incident note (2026-11-04).
|
|
2724
3205
|
public static ClipboardLease Begin(IClipboardAccess access, string text)
|
|
2725
3206
|
{
|
|
2726
3207
|
using (var session = access.Open())
|
|
2727
3208
|
{
|
|
2728
3209
|
if (session.Sequence == 0) throw new Exception("Clipboard sequence unavailable; clipboard was not changed.");
|
|
2729
|
-
var previous = session.Snapshot(); //
|
|
3210
|
+
var previous = session.Snapshot(); // null = cannot preserve: caller downgrades, we do not clear anything here
|
|
3211
|
+
if (previous == null) return null;
|
|
2730
3212
|
try
|
|
2731
3213
|
{
|
|
2732
3214
|
session.Replace(new List<ClipboardBytes> {
|
|
@@ -2765,8 +3247,9 @@ namespace DshComputerUse.Worker
|
|
|
2765
3247
|
}
|
|
2766
3248
|
}
|
|
2767
3249
|
|
|
2768
|
-
// Copy only explicitly supported HGLOBAL formats. Unknown/private or GDI formats
|
|
2769
|
-
//
|
|
3250
|
+
// Copy only explicitly supported HGLOBAL formats. Unknown/private or GDI formats cannot be
|
|
3251
|
+
// snapshotted — Snapshot returns null for them and the caller downgrades to
|
|
3252
|
+
// paste-without-restore with a warning (incident 2026-11-04) instead of losing the paste.
|
|
2770
3253
|
public sealed class NativeClipboardAccess : IClipboardAccess, IDisposable
|
|
2771
3254
|
{
|
|
2772
3255
|
readonly NativeWindow window = new NativeWindow();
|
|
@@ -2790,14 +3273,6 @@ namespace DshComputerUse.Worker
|
|
|
2790
3273
|
[DllImport("kernel32.dll")] static extern UIntPtr GlobalSize(IntPtr memory);
|
|
2791
3274
|
[DllImport("kernel32.dll")] static extern IntPtr GlobalFree(IntPtr memory);
|
|
2792
3275
|
[DllImport("kernel32.dll")] static extern void SetLastError(uint error);
|
|
2793
|
-
static Exception Unavailable(string reason)
|
|
2794
|
-
{
|
|
2795
|
-
var error = new Exception("CLIPBOARD_PRESERVATION_UNAVAILABLE: " + reason +
|
|
2796
|
-
"; clipboard unchanged. Use mode:unicode if suitable for the target editor.");
|
|
2797
|
-
error.Data["code"] = "CLIPBOARD_PRESERVATION_UNAVAILABLE";
|
|
2798
|
-
error.Data["outcome"] = "not-dispatched";
|
|
2799
|
-
return error;
|
|
2800
|
-
}
|
|
2801
3276
|
public static bool Supported(uint format, string name)
|
|
2802
3277
|
{
|
|
2803
3278
|
if (format == 1 || format == 7 || format == 8 || format == 13 || format == 15 || format == 16 || format == 17) return true;
|
|
@@ -2819,6 +3294,13 @@ namespace DshComputerUse.Worker
|
|
|
2819
3294
|
}
|
|
2820
3295
|
public uint Sequence { get { return GetClipboardSequenceNumber(); } }
|
|
2821
3296
|
public void Dispose() { if (opened) { CloseClipboard(); opened = false; } }
|
|
3297
|
+
/// Returns null when the current clipboard cannot be fully snapshotted (private or GDI
|
|
3298
|
+
/// formats, too many/oversized entries, lock failures). NULL MEANS "cannot preserve",
|
|
3299
|
+
/// not "refuse to work" (human incident 2026-11-04): a WeChat-session clipboard carried
|
|
3300
|
+
/// private format 49161, paste was refused, the JS auto-fallback re-typed 37 CJK chars
|
|
3301
|
+
/// as unicode keystrokes and a garbled message actually went out. Losing an opaque
|
|
3302
|
+
/// clipboard item is a recoverable local cost; a mis-typed external message is not —
|
|
3303
|
+
/// so the caller downgrades to paste-without-restore instead of refusing.
|
|
2822
3304
|
public List<ClipboardBytes> Snapshot()
|
|
2823
3305
|
{
|
|
2824
3306
|
var formats = new List<uint>();
|
|
@@ -2829,27 +3311,26 @@ namespace DshComputerUse.Worker
|
|
|
2829
3311
|
format = EnumClipboardFormats(format);
|
|
2830
3312
|
if (format == 0)
|
|
2831
3313
|
{
|
|
2832
|
-
if (Marshal.GetLastWin32Error() != 0)
|
|
3314
|
+
if (Marshal.GetLastWin32Error() != 0) return null;
|
|
2833
3315
|
break;
|
|
2834
3316
|
}
|
|
2835
3317
|
var name = new StringBuilder(256);
|
|
2836
3318
|
if (format >= 0xC000) GetClipboardFormatName(format, name, name.Capacity);
|
|
2837
|
-
if (!Supported(format, name.ToString()))
|
|
3319
|
+
if (!Supported(format, name.ToString())) return null;
|
|
2838
3320
|
formats.Add(format);
|
|
2839
|
-
if (formats.Count > 64)
|
|
3321
|
+
if (formats.Count > 64) return null;
|
|
2840
3322
|
}
|
|
2841
3323
|
var result = new List<ClipboardBytes>();
|
|
2842
3324
|
long total = 0;
|
|
2843
3325
|
foreach (uint id in formats)
|
|
2844
3326
|
{
|
|
2845
3327
|
IntPtr handle = GetClipboardData(id);
|
|
2846
|
-
if (handle == IntPtr.Zero)
|
|
3328
|
+
if (handle == IntPtr.Zero) return null;
|
|
2847
3329
|
ulong size = GlobalSize(handle).ToUInt64();
|
|
2848
|
-
if (size == 0 || size > 32 * 1024 * 1024 || total + (long)size > 32 * 1024 * 1024)
|
|
2849
|
-
throw Unavailable("clipboard snapshot exceeds supported size");
|
|
3330
|
+
if (size == 0 || size > 32 * 1024 * 1024 || total + (long)size > 32 * 1024 * 1024) return null;
|
|
2850
3331
|
total += (long)size;
|
|
2851
3332
|
IntPtr data = GlobalLock(handle);
|
|
2852
|
-
if (data == IntPtr.Zero)
|
|
3333
|
+
if (data == IntPtr.Zero) return null;
|
|
2853
3334
|
try
|
|
2854
3335
|
{
|
|
2855
3336
|
var bytes = new byte[(int)size]; Marshal.Copy(data, bytes, 0, bytes.Length);
|
|
@@ -2890,6 +3371,16 @@ namespace DshComputerUse.Worker
|
|
|
2890
3371
|
|
|
2891
3372
|
static class Program
|
|
2892
3373
|
{
|
|
3374
|
+
public const int ProtocolVersion = 2;
|
|
3375
|
+
public static readonly string WorkerGeneration = Guid.NewGuid().ToString("N");
|
|
3376
|
+
static Dictionary<string, object> RuntimeIdentity()
|
|
3377
|
+
{
|
|
3378
|
+
return Dict("packageVersion", Environment.GetEnvironmentVariable("DSH_CU_PACKAGE_VERSION") ?? "unreported",
|
|
3379
|
+
"buildId", Environment.GetEnvironmentVariable("DSH_CU_BUILD_ID") ?? "unreported",
|
|
3380
|
+
"sourceHash", Environment.GetEnvironmentVariable("DSH_CU_SOURCE_HASH") ?? "unreported",
|
|
3381
|
+
"binaryHash", Environment.GetEnvironmentVariable("DSH_CU_BINARY_HASH") ?? "unreported",
|
|
3382
|
+
"protocolVersion", ProtocolVersion, "workerGeneration", WorkerGeneration);
|
|
3383
|
+
}
|
|
2893
3384
|
static readonly JavaScriptSerializer Json = new JavaScriptSerializer();
|
|
2894
3385
|
static int _reqId = 0;
|
|
2895
3386
|
static readonly object OutLock = new object(); // responses AND unsolicited events share stdout
|
|
@@ -2901,7 +3392,7 @@ namespace DshComputerUse.Worker
|
|
|
2901
3392
|
static string _dpiNote = "unset";
|
|
2902
3393
|
static readonly HashSet<string> ObsOps = new HashSet<string>
|
|
2903
3394
|
{
|
|
2904
|
-
"ping", "echoargs", "screens", "cursor", "capture", "annotate",
|
|
3395
|
+
"ping", "echoargs", "screens", "cursor", "capture", "ocr", "annotate",
|
|
2905
3396
|
"windows", "uia", "uiaFromPoint", "uiaFocused", "waitForIdle", "clipRead",
|
|
2906
3397
|
"selftest", "indicator", "waitStable", "waitChange", "focus", "frameSig", "probe"
|
|
2907
3398
|
};
|
|
@@ -2952,6 +3443,33 @@ namespace DshComputerUse.Worker
|
|
|
2952
3443
|
return Native.SendInput((uint)inputs.Length, inputs, InputCbSize);
|
|
2953
3444
|
};
|
|
2954
3445
|
internal static Func<IntPtr> PreparationForeground = delegate { return Native.GetForegroundWindow(); };
|
|
3446
|
+
internal static Func<IntPtr> InputForeground = delegate { return TopLevel(Native.GetForegroundWindow()); };
|
|
3447
|
+
internal static Func<IntPtr> InputPointReceiver = delegate {
|
|
3448
|
+
POINT p; return Native.GetCursorPos(out p) ? TopLevel(Native.WindowFromPoint(p)) : IntPtr.Zero;
|
|
3449
|
+
};
|
|
3450
|
+
internal static Func<IntPtr, bool> InputWindowExists = delegate(IntPtr h) { return Native.IsWindow(h); };
|
|
3451
|
+
internal static Func<IntPtr, uint> InputWindowPid = delegate(IntPtr h) { return PidOf(h); };
|
|
3452
|
+
[ThreadStatic] static Action _receiverCheck;
|
|
3453
|
+
sealed class ReceiverScope : IDisposable
|
|
3454
|
+
{
|
|
3455
|
+
readonly Action previous;
|
|
3456
|
+
public ReceiverScope(Action check) { previous = _receiverCheck; check(); _receiverCheck = check; }
|
|
3457
|
+
public void Dispose() { _receiverCheck = previous; }
|
|
3458
|
+
}
|
|
3459
|
+
static string InputReceiverProblem(long expected, uint expectedPid, bool exists, uint livePid, long actual)
|
|
3460
|
+
{
|
|
3461
|
+
if (expected == 0 || !exists || livePid != expectedPid) return "observed target disappeared or changed identity";
|
|
3462
|
+
if (actual != expected) return "input receiver changed from hwnd=" + expected + " to hwnd=" + actual;
|
|
3463
|
+
return null;
|
|
3464
|
+
}
|
|
3465
|
+
static IDisposable GuardReceiver(IntPtr expected, uint pid, bool keyboard)
|
|
3466
|
+
{
|
|
3467
|
+
return new ReceiverScope(delegate {
|
|
3468
|
+
IntPtr actual = keyboard ? InputForeground() : InputPointReceiver();
|
|
3469
|
+
string problem = InputReceiverProblem(expected.ToInt64(), pid, InputWindowExists(expected), InputWindowPid(expected), actual.ToInt64());
|
|
3470
|
+
if (problem != null) throw InputInterrupted("INPUT_TARGET_CHANGED", problem + ". Remaining input was refused; inspect any partial effect before retrying.");
|
|
3471
|
+
});
|
|
3472
|
+
}
|
|
2955
3473
|
|
|
2956
3474
|
static Exception InputInterrupted(string code, string message)
|
|
2957
3475
|
{
|
|
@@ -2962,6 +3480,8 @@ namespace DshComputerUse.Worker
|
|
|
2962
3480
|
}
|
|
2963
3481
|
public static void CheckHumanInput()
|
|
2964
3482
|
{
|
|
3483
|
+
if (Panic.ExitRequested)
|
|
3484
|
+
throw InputInterrupted("CYCLE-ENDED", "Ctrl+Alt+Q requested exit; no further input will be sent for this operation.");
|
|
2965
3485
|
if (Panic.PauseRequested)
|
|
2966
3486
|
throw InputInterrupted("ABORTED", "Ctrl+Esc requested a manual pause; wait for manual recovery.");
|
|
2967
3487
|
if (_operationLive && _operationCancelGeneration != Interlocked.Read(ref _cancelGeneration))
|
|
@@ -3017,16 +3537,16 @@ namespace DshComputerUse.Worker
|
|
|
3017
3537
|
while (!Glow.PreparationVisible)
|
|
3018
3538
|
{
|
|
3019
3539
|
Panic.Check(op); CheckHumanInput();
|
|
3020
|
-
if (startup.ElapsedMilliseconds >= 5000)
|
|
3021
|
-
throw InputInterrupted("PREPARATION_UNAVAILABLE", "The preparation notice could not be displayed; no input was dispatched.");
|
|
3540
|
+
if (startup.ElapsedMilliseconds >= 5000) throw PreparationUnavailable("first-frame");
|
|
3022
3541
|
Thread.Sleep(10);
|
|
3023
3542
|
}
|
|
3024
3543
|
var timer = Stopwatch.StartNew();
|
|
3025
|
-
while (timer.ElapsedMilliseconds < Glow.PreparationMs)
|
|
3544
|
+
while (timer.ElapsedMilliseconds < Glow.PreparationMs || !Glow.PreparationComplete)
|
|
3026
3545
|
{
|
|
3027
3546
|
Panic.Check(op); CheckHumanInput();
|
|
3547
|
+
if (timer.ElapsedMilliseconds >= 5000) throw PreparationUnavailable("countdown-frame");
|
|
3028
3548
|
int remaining = Glow.PreparationMs - (int)timer.ElapsedMilliseconds;
|
|
3029
|
-
|
|
3549
|
+
Thread.Sleep(remaining > 0 ? Math.Min(25, remaining) : 10);
|
|
3030
3550
|
}
|
|
3031
3551
|
Panic.Check(op); CheckHumanInput();
|
|
3032
3552
|
if (inputEpoch != Panic.InputEpoch || foreground != PreparationForeground())
|
|
@@ -3036,13 +3556,41 @@ namespace DshComputerUse.Worker
|
|
|
3036
3556
|
}
|
|
3037
3557
|
finally { Glow.Prepare(false); }
|
|
3038
3558
|
}
|
|
3559
|
+
static Exception PreparationUnavailable(string phase)
|
|
3560
|
+
{
|
|
3561
|
+
var health = Glow.Health();
|
|
3562
|
+
health["failurePhase"] = phase;
|
|
3563
|
+
var error = InputInterrupted("PREPARATION_UNAVAILABLE", "The preparation notice could not be displayed through completion; no input was dispatched. glowHealth=" + Json.Serialize(health));
|
|
3564
|
+
error.Data["glowHealth"] = health;
|
|
3565
|
+
return error;
|
|
3566
|
+
}
|
|
3039
3567
|
public static bool CancelInput(int requestId)
|
|
3040
3568
|
{
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3569
|
+
return GetB(CancelInputStatus(requestId), "cancelled", false);
|
|
3570
|
+
}
|
|
3571
|
+
static Dictionary<string, object> CancelInputStatus(int requestId)
|
|
3572
|
+
{
|
|
3573
|
+
bool valid = requestId > 0;
|
|
3574
|
+
bool completed = valid && requestId <= _lastCompletedRequestId;
|
|
3575
|
+
if (valid && !completed) CancelledRequests[requestId] = true;
|
|
3576
|
+
bool active = valid && _activeRequestId == requestId;
|
|
3577
|
+
if (active) Interlocked.Increment(ref _cancelGeneration);
|
|
3578
|
+
bool cleanup = false; int owned = -1; string state = valid ? "input-lock-busy" : "invalid-request";
|
|
3579
|
+
// The reader must remain responsive even if SendInput/provider work
|
|
3580
|
+
// owns InputLock. Failure to obtain it is uncertainty, not clean-up.
|
|
3581
|
+
if (valid && Monitor.TryEnter(InputLock, 200)) {
|
|
3582
|
+
try {
|
|
3583
|
+
// Never release a newer request's accepted input on behalf
|
|
3584
|
+
// of an older cancellation.
|
|
3585
|
+
if (_activeRequestId == requestId) ReleaseOwnedInputs();
|
|
3586
|
+
owned = OwnedUps.Count;
|
|
3587
|
+
cleanup = owned == 0;
|
|
3588
|
+
state = cleanup ? "no-owned-input" : "owned-input-remains";
|
|
3589
|
+
} finally { Monitor.Exit(InputLock); }
|
|
3590
|
+
}
|
|
3591
|
+
return Dict("requestId", requestId, "cancelled", valid && !completed,
|
|
3592
|
+
"cleanupConfirmed", cleanup, "ownedInputCount", owned, "cleanupState", state,
|
|
3593
|
+
"workerGeneration", WorkerGeneration);
|
|
3046
3594
|
}
|
|
3047
3595
|
static string InputKey(INPUT input, out bool up, out INPUT release, out int physical)
|
|
3048
3596
|
{
|
|
@@ -3083,6 +3631,8 @@ namespace DshComputerUse.Worker
|
|
|
3083
3631
|
string key = InputKey(inputs[0], out isUp, out release, out physical);
|
|
3084
3632
|
if (isUp && key != null && !OwnedUps.ContainsKey(key)) return 1;
|
|
3085
3633
|
}
|
|
3634
|
+
if (_receiverCheck != null) _receiverCheck();
|
|
3635
|
+
CheckHumanInput();
|
|
3086
3636
|
uint sent = InputSender(inputs);
|
|
3087
3637
|
if (sent > 0) _operationDispatched = true;
|
|
3088
3638
|
for (int i = 0; i < Math.Min(sent, inputs.Length); i++)
|
|
@@ -3226,6 +3776,16 @@ namespace DshComputerUse.Worker
|
|
|
3226
3776
|
[STAThread]
|
|
3227
3777
|
static int Main(string[] args)
|
|
3228
3778
|
{
|
|
3779
|
+
// The cursor guardian owns only the temporary visual-hide lease. It
|
|
3780
|
+
// must never install input hooks, open a control cycle or emit worker
|
|
3781
|
+
// protocol messages into the parent's inherited output stream.
|
|
3782
|
+
if (args.Length > 0 && args[0] == "--pointer-guardian")
|
|
3783
|
+
{
|
|
3784
|
+
int ownerPid; long ownerStartTicks;
|
|
3785
|
+
if (args.Length != 4 || !Int32.TryParse(args[2], out ownerPid) ||
|
|
3786
|
+
!Int64.TryParse(args[3], out ownerStartTicks)) return 2;
|
|
3787
|
+
return PointerVisibilityLease.GuardianMain(args[1], ownerPid, ownerStartTicks);
|
|
3788
|
+
}
|
|
3229
3789
|
// DPI awareness first: it decides the coordinate space of everything below
|
|
3230
3790
|
// (GetSystemMetrics, GetCursorPos, capture pixels, UIA rects AND the
|
|
3231
3791
|
// interpretation of SendInput's absolute coordinates). Record which call won.
|
|
@@ -3254,8 +3814,9 @@ namespace DshComputerUse.Worker
|
|
|
3254
3814
|
// lives; the consumer can read the fields and draw no conclusion the worker did
|
|
3255
3815
|
// not record itself.
|
|
3256
3816
|
string hint;
|
|
3257
|
-
|
|
3258
|
-
|
|
3817
|
+
string pauseId = Panic.EventPauseId(kind);
|
|
3818
|
+
if (kind == "panic" && !string.IsNullOrEmpty(pauseId))
|
|
3819
|
+
hint = "Agent temporary diagnostic pause. Inspect the current target, then use checked recover only if the blocker is resolved. Credential: " + pauseId;
|
|
3259
3820
|
else if (kind == "panic")
|
|
3260
3821
|
hint = "STOPPED (brake engaged). The machine is stopped: nothing that drives it runs " +
|
|
3261
3822
|
"until a human resumes with Ctrl+Alt+R. WRAP UP NOW: one short summary of what " +
|
|
@@ -3271,12 +3832,16 @@ namespace DshComputerUse.Worker
|
|
|
3271
3832
|
else if (kind == "resume")
|
|
3272
3833
|
hint = "Re-armed: a human pressed Ctrl+Alt+R (the worker notifies `resume` from Clear() " +
|
|
3273
3834
|
"alone) — working may continue from the break point.";
|
|
3835
|
+
else if (kind == "secure")
|
|
3836
|
+
hint = why == "active"
|
|
3837
|
+
? "The Windows secure desktop is up (a UAC prompt or the lock screen). Input dispatch is PAUSED — nothing you send will be dispatched until it closes. Tell the human a system dialog needs their answer; dispatch resumes automatically the moment the normal desktop returns, then simply retry the step."
|
|
3838
|
+
: "The normal desktop is back. Input dispatch has resumed automatically — retry the interrupted step and continue the task.";
|
|
3274
3839
|
else
|
|
3275
3840
|
hint = "worker state change \"" + kind + "\": " + why;
|
|
3276
3841
|
WriteLine(Json.Serialize(Dict(
|
|
3277
3842
|
"event", kind,
|
|
3278
3843
|
"why", why,
|
|
3279
|
-
"pauseId",
|
|
3844
|
+
"pauseId", pauseId,
|
|
3280
3845
|
"cooperation", Panic.CooperationState(),
|
|
3281
3846
|
"ts", NowMs(),
|
|
3282
3847
|
"hint", hint)));
|
|
@@ -3301,6 +3866,11 @@ namespace DshComputerUse.Worker
|
|
|
3301
3866
|
Dictionary<string, object> resp;
|
|
3302
3867
|
try { resp = Handle(oneshot); }
|
|
3303
3868
|
catch (Exception ex) { resp = Err(ex); }
|
|
3869
|
+
if (!Panic.FinishOneShotControls())
|
|
3870
|
+
{
|
|
3871
|
+
Program.AuditRecord("CONTROL-DRAIN-TIMEOUT", "one-shot control finalization exceeded 5s");
|
|
3872
|
+
resp = Err("CONTROL-DRAIN-TIMEOUT: physical brake finalization could not be confirmed; verify STOP/EXITED before continuing.");
|
|
3873
|
+
}
|
|
3304
3874
|
AuditActuation(oneshot, resp);
|
|
3305
3875
|
WriteLine(Json.Serialize(resp));
|
|
3306
3876
|
return 0;
|
|
@@ -3308,6 +3878,8 @@ namespace DshComputerUse.Worker
|
|
|
3308
3878
|
Dictionary<string, object> hello = Dict(
|
|
3309
3879
|
"event", "ready",
|
|
3310
3880
|
"pid", Process.GetCurrentProcess().Id,
|
|
3881
|
+
"protocolVersion", ProtocolVersion, "workerGeneration", WorkerGeneration,
|
|
3882
|
+
"runtime", RuntimeIdentity(),
|
|
3311
3883
|
// COLD-START STATE, REPORTED AS A FACT (cold-start bug, 2026-09-13): this process has
|
|
3312
3884
|
// already adopted any STOP/EXITED record that was on disk before it existed, so the
|
|
3313
3885
|
// core can start from the machine's real state instead of "not stopped" — the two
|
|
@@ -3332,9 +3904,9 @@ namespace DshComputerUse.Worker
|
|
|
3332
3904
|
{
|
|
3333
3905
|
object argsValue;
|
|
3334
3906
|
var cancelArgs = request.TryGetValue("args", out argsValue) ? argsValue as Dictionary<string, object> : null;
|
|
3335
|
-
|
|
3907
|
+
var cancellation = CancelInputStatus(cancelArgs == null ? 0 : GetI(cancelArgs, "requestId", 0));
|
|
3336
3908
|
WriteLine(parser.Serialize(Dict("id", GetI(request, "id", 0), "ok", true,
|
|
3337
|
-
"data",
|
|
3909
|
+
"data", cancellation)));
|
|
3338
3910
|
continue;
|
|
3339
3911
|
}
|
|
3340
3912
|
} catch { /* Main loop reports malformed requests in the normal format. */ }
|
|
@@ -3387,6 +3959,11 @@ namespace DshComputerUse.Worker
|
|
|
3387
3959
|
var args = request.TryGetValue("args", out rawArgs) ? rawArgs as Dictionary<string, object> : null;
|
|
3388
3960
|
bool input = ActOps.Contains(op) || op == "shiftClick" || op == "selftest" ||
|
|
3389
3961
|
(op == "focus" && args != null && GetB(args, "refocus", false));
|
|
3962
|
+
bool dryInput = input && args != null && GetB(args, "dryRun", false);
|
|
3963
|
+
// Only the coordinate click probe has a native read-only implementation.
|
|
3964
|
+
// No other input may use the flag to skip the controls and then dispatch.
|
|
3965
|
+
if (dryInput && (op != "click" || !args.ContainsKey("x") || !args.ContainsKey("y")))
|
|
3966
|
+
return Err("UNSUPPORTED_DRY_RUN: native input simulation requires a click with x and y; no input was dispatched.");
|
|
3390
3967
|
EnsureDesktopOwnership(op);
|
|
3391
3968
|
int requestId = GetI(request, "id", 0);
|
|
3392
3969
|
bool ignored;
|
|
@@ -3394,13 +3971,14 @@ namespace DshComputerUse.Worker
|
|
|
3394
3971
|
_operationHumanRevision = Panic.HumanRevision;
|
|
3395
3972
|
_operationCancelGeneration = Interlocked.Read(ref _cancelGeneration);
|
|
3396
3973
|
_operationDispatched = false; _operationLive = input;
|
|
3974
|
+
bool scopedCycle = false;
|
|
3397
3975
|
try
|
|
3398
3976
|
{
|
|
3399
3977
|
if (CancelledRequests.TryRemove(requestId, out ignored))
|
|
3400
3978
|
throw InputInterrupted("INPUT_CANCELLED", "cancelled before dispatch.");
|
|
3401
3979
|
if (input) CheckHumanInput();
|
|
3402
3980
|
string pauseId = Panic.TemporaryPauseId;
|
|
3403
|
-
Dictionary<string, object> result = HandleCore(line);
|
|
3981
|
+
Dictionary<string, object> result = HandleCore(line, input, dryInput, ref scopedCycle);
|
|
3404
3982
|
if (input) CheckHumanInput();
|
|
3405
3983
|
if (!string.IsNullOrEmpty(pauseId) && GetB(result, "ok", false) &&
|
|
3406
3984
|
(op == "windows" || op == "uia" || op == "uiaFromPoint" || op == "uiaFocused" || op == "capture"))
|
|
@@ -3415,13 +3993,14 @@ namespace DshComputerUse.Worker
|
|
|
3415
3993
|
finally { _operationActing = false; Panic.EndActuation(); Glow.Prepare(false); }
|
|
3416
3994
|
}
|
|
3417
3995
|
_operationLive = false; _activeRequestId = 0;
|
|
3996
|
+
if (scopedCycle) Panic.TryEndCycle();
|
|
3418
3997
|
if (_releaseDesktopPending) ReleaseDesktopOwnership();
|
|
3419
3998
|
_lastCompletedRequestId = Math.Max(_lastCompletedRequestId, requestId);
|
|
3420
3999
|
CancelledRequests.TryRemove(requestId, out ignored);
|
|
3421
4000
|
}
|
|
3422
4001
|
}
|
|
3423
4002
|
|
|
3424
|
-
static Dictionary<string, object> HandleCore(string line)
|
|
4003
|
+
static Dictionary<string, object> HandleCore(string line, bool input, bool dryInput, ref bool scopedCycle)
|
|
3425
4004
|
{
|
|
3426
4005
|
Dictionary<string, object> req;
|
|
3427
4006
|
try { req = Json.Deserialize<Dictionary<string, object>>(line); }
|
|
@@ -3454,11 +4033,22 @@ namespace DshComputerUse.Worker
|
|
|
3454
4033
|
// human re-arms it (CheckOp's allowlist keeps the health probe and re-arm alive).
|
|
3455
4034
|
// Emergency stop gates EVERY op that could drive the machine; a DRY RUN drives nothing
|
|
3456
4035
|
// and is therefore treated as the observation it really is.
|
|
3457
|
-
Panic.CheckOp(
|
|
4036
|
+
Panic.CheckOp(dryInput ? "probe" : op);
|
|
3458
4037
|
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
4038
|
+
if (input && !dryInput)
|
|
4039
|
+
{
|
|
4040
|
+
// Direct native input (including --op) has no JS cycle signature.
|
|
4041
|
+
// Arm physical controls before the purple cue and release this scope
|
|
4042
|
+
// when the request ends. A human pause or exit retains its own latch.
|
|
4043
|
+
if (!Panic.CycleLit)
|
|
4044
|
+
{
|
|
4045
|
+
scopedCycle = Panic.TryLightCycle(0);
|
|
4046
|
+
if (!scopedCycle) Panic.Check(op);
|
|
4047
|
+
}
|
|
4048
|
+
Panic.RequireControlHooks();
|
|
4049
|
+
Panic.RequireNormalDesktop();
|
|
4050
|
+
PrepareActuation(op, req);
|
|
4051
|
+
}
|
|
3462
4052
|
|
|
3463
4053
|
switch (op)
|
|
3464
4054
|
{
|
|
@@ -3470,7 +4060,7 @@ namespace DshComputerUse.Worker
|
|
|
3470
4060
|
case "beginAsk": return Ok(BeginAskOp(a));
|
|
3471
4061
|
case "endAsk": return Ok(EndAskOp(a));
|
|
3472
4062
|
case "exit": return Ok(ExitOp(a));
|
|
3473
|
-
case "ping": return Ok(Dict("pong", true, "cooperation", Panic.CooperationState()));
|
|
4063
|
+
case "ping": return Ok(Dict("pong", true, "cooperation", Panic.CooperationState(), "glowHealth", Glow.Health()));
|
|
3474
4064
|
// TEST SEAM (D, 2026-09-13). A fixture may not re-implement the rule it measures, so
|
|
3475
4065
|
// this op drives the PRODUCTION transition (Panic.ClassifyAndTrackKey — the same
|
|
3476
4066
|
// function the low-level hook calls) with explicit evidence: ours / injected / vk.
|
|
@@ -3536,6 +4126,7 @@ namespace DshComputerUse.Worker
|
|
|
3536
4126
|
case "screens": return Ok(VirtualScreenDict());
|
|
3537
4127
|
case "cursor": POINT cp; Native.GetCursorPos(out cp); return Ok(PtDict(cp.X, cp.Y));
|
|
3538
4128
|
case "capture": return Ok(Capture(a));
|
|
4129
|
+
case "ocr": return Ok(Ocr(a));
|
|
3539
4130
|
case "annotate": return Ok(Annotate(a));
|
|
3540
4131
|
case "move": FailsafeCheck(a); return Ok(Move(a));
|
|
3541
4132
|
case "click": FailsafeCheck(a); return Ok(Click(a));
|
|
@@ -3557,13 +4148,15 @@ namespace DshComputerUse.Worker
|
|
|
3557
4148
|
case "probe":
|
|
3558
4149
|
{
|
|
3559
4150
|
string what = GetS(a, "what", "point");
|
|
3560
|
-
Dictionary<string, object> pr = what == "focus" ? ProbeFocus()
|
|
4151
|
+
Dictionary<string, object> pr = what == "glow" ? Dict() : what == "focus" ? ProbeFocus()
|
|
3561
4152
|
: (what == "window" ? WinBrief(FindHwnd(a)) : ProbePoint(GetI(a, "x", 0), GetI(a, "y", 0)));
|
|
4153
|
+
pr["glowHealth"] = Glow.Health();
|
|
3562
4154
|
// The state machine as a FACT, in the one op that exists to answer "what is
|
|
3563
4155
|
// actually true right now". Read-only, so asking never changes the answer.
|
|
3564
4156
|
pr["cycle"] = Dict(
|
|
3565
4157
|
"cycleLit", Panic.CycleLit,
|
|
3566
4158
|
"armed", Panic.Armed,
|
|
4159
|
+
"secureDesktop", Panic.SecureDesktopActive,
|
|
3567
4160
|
"engaged", Panic.Engaged,
|
|
3568
4161
|
"stopped", Panic.Stopped,
|
|
3569
4162
|
"exited", Panic.Exited,
|
|
@@ -3580,7 +4173,7 @@ namespace DshComputerUse.Worker
|
|
|
3580
4173
|
// Is a real injection sequence in flight RIGHT NOW? The ACTING light is driven
|
|
3581
4174
|
// by the actual action scope, so both are reported as observable facts.
|
|
3582
4175
|
"injecting", Glow.Injecting,
|
|
3583
|
-
"acting", Glow.Acting, "preparing", Glow.Preparing, "preparationMs", Glow.PreparationMs,
|
|
4176
|
+
"acting", Glow.Acting, "holding", Glow.Holding, "preparing", Glow.Preparing, "preparationMs", Glow.PreparationMs,
|
|
3584
4177
|
"title", Glow.ShownTitle == null ? "" : Glow.ShownTitle);
|
|
3585
4178
|
// EVENTS THE OS ACTUALLY INSERTED (not the plan): 0 after a plain pause with
|
|
3586
4179
|
// nothing held is the observable form of "no ghost right-click / Alt menu".
|
|
@@ -3645,6 +4238,90 @@ namespace DshComputerUse.Worker
|
|
|
3645
4238
|
|
|
3646
4239
|
static long NowMs() { return (long)(DateTime.UtcNow - Epoch).TotalMilliseconds; }
|
|
3647
4240
|
|
|
4241
|
+
// ---------- OCR: text WITH bounding boxes, straight from the worker ----------
|
|
4242
|
+
// Zero-UIA apps (WeChat incident 2026-11-04) burned one screenshot per decision because
|
|
4243
|
+
// finding "傳送" or a contact name meant reading pixels with model vision. Windows' inbox
|
|
4244
|
+
// OCR (Windows.Media.Ocr — ships with the OS, zero external dependencies) returns text
|
|
4245
|
+
// with per-word rectangles, so the model can locate controls BY NAME and spend vision
|
|
4246
|
+
// only where looks actually matter. Coordinates are ABSOLUTE screen pixels (region origin
|
|
4247
|
+
// added), ready for computer_click without further math.
|
|
4248
|
+
static OcrEngine _ocrEngine;
|
|
4249
|
+
static string _ocrLanguage;
|
|
4250
|
+
static Dictionary<string, object> Ocr(Dictionary<string, object> a)
|
|
4251
|
+
{
|
|
4252
|
+
ValidateKickoffCapture(a);
|
|
4253
|
+
if (_ocrEngine == null)
|
|
4254
|
+
{
|
|
4255
|
+
_ocrEngine = OcrEngine.TryCreateFromUserProfileLanguages();
|
|
4256
|
+
if (_ocrEngine != null) _ocrLanguage = _ocrEngine.RecognizerLanguage.LanguageTag;
|
|
4257
|
+
}
|
|
4258
|
+
if (_ocrEngine == null)
|
|
4259
|
+
throw new Exception("OCR_UNAVAILABLE: no OCR recognizer language is installed on this machine. " +
|
|
4260
|
+
"Install one (Settings > Time & language > Language > add a language with 'Text recognition') " +
|
|
4261
|
+
"or read regions with computer_shot instead.");
|
|
4262
|
+
int vx = Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
4263
|
+
int vy = Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
4264
|
+
int vw = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
4265
|
+
int vh = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
4266
|
+
int x = GetI(a, "x", vx), y = GetI(a, "y", vy);
|
|
4267
|
+
int w = GetI(a, "width", vw), h = GetI(a, "height", vh);
|
|
4268
|
+
if (GetB(a, "clamp", true))
|
|
4269
|
+
{
|
|
4270
|
+
int x2 = Math.Max(x, vx), y2 = Math.Max(y, vy);
|
|
4271
|
+
int x3 = Math.Min(x + w, vx + vw), y3 = Math.Min(y + h, vy + vh);
|
|
4272
|
+
if (x3 <= x2 || y3 <= y2) throw new Exception("region outside virtual desktop");
|
|
4273
|
+
x = x2; y = y2; w = x3 - x2; h = y3 - y2;
|
|
4274
|
+
}
|
|
4275
|
+
// OCR cost scales with pixels; one decision never needs more than 16M of them.
|
|
4276
|
+
if ((long)w * h > 16L * 1024 * 1024)
|
|
4277
|
+
throw new Exception("OCR region exceeds 16M pixels (" + w + "x" + h + "); crop tighter.");
|
|
4278
|
+
byte[] bgra = new byte[w * h * 4];
|
|
4279
|
+
IntPtr screen = IntPtr.Zero, mem = IntPtr.Zero, dib = IntPtr.Zero, oldBmp = IntPtr.Zero;
|
|
4280
|
+
try
|
|
4281
|
+
{
|
|
4282
|
+
screen = Native.GetDC(IntPtr.Zero);
|
|
4283
|
+
mem = Native.CreateCompatibleDC(screen);
|
|
4284
|
+
Native.BITMAPINFO bmi = new Native.BITMAPINFO();
|
|
4285
|
+
bmi.bmiHeader.biSize = (uint)Marshal.SizeOf(typeof(Native.BITMAPINFOHEADER));
|
|
4286
|
+
bmi.bmiHeader.biWidth = w; bmi.bmiHeader.biHeight = -h; // top-down
|
|
4287
|
+
bmi.bmiHeader.biPlanes = 1; bmi.bmiHeader.biBitCount = 32;
|
|
4288
|
+
bmi.bmiHeader.biCompression = 0;
|
|
4289
|
+
IntPtr bits;
|
|
4290
|
+
dib = Native.CreateDIBSection(screen, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
4291
|
+
if (dib == IntPtr.Zero) throw new Exception("OCR: CreateDIBSection failed");
|
|
4292
|
+
oldBmp = Native.SelectObject(mem, dib);
|
|
4293
|
+
Native.BitBlt(mem, 0, 0, w, h, screen, x, y, 0x00CC0020 | 0x40000000);
|
|
4294
|
+
Marshal.Copy(bits, bgra, 0, bgra.Length);
|
|
4295
|
+
}
|
|
4296
|
+
finally
|
|
4297
|
+
{
|
|
4298
|
+
if (oldBmp != IntPtr.Zero && oldBmp != new IntPtr(-1)) Native.SelectObject(mem, oldBmp);
|
|
4299
|
+
if (dib != IntPtr.Zero) Native.DeleteObject(dib);
|
|
4300
|
+
if (mem != IntPtr.Zero) Native.DeleteDC(mem);
|
|
4301
|
+
if (screen != IntPtr.Zero) Native.ReleaseDC(IntPtr.Zero, screen);
|
|
4302
|
+
}
|
|
4303
|
+
SoftwareBitmap bitmap = SoftwareBitmap.CreateCopyFromBuffer(
|
|
4304
|
+
WindowsRuntimeBufferExtensions.AsBuffer(bgra), BitmapPixelFormat.Bgra8, w, h);
|
|
4305
|
+
OcrResult result = _ocrEngine.RecognizeAsync(bitmap).AsTask().GetAwaiter().GetResult();
|
|
4306
|
+
List<object> lines = new List<object>();
|
|
4307
|
+
StringBuilder joined = new StringBuilder();
|
|
4308
|
+
foreach (OcrLine line in result.Lines)
|
|
4309
|
+
{
|
|
4310
|
+
List<object> words = new List<object>();
|
|
4311
|
+
foreach (OcrWord word in line.Words)
|
|
4312
|
+
{
|
|
4313
|
+
words.Add(Dict("text", word.Text,
|
|
4314
|
+
"x", x + (int)word.BoundingRect.X, "y", y + (int)word.BoundingRect.Y,
|
|
4315
|
+
"w", (int)word.BoundingRect.Width, "h", (int)word.BoundingRect.Height));
|
|
4316
|
+
}
|
|
4317
|
+
lines.Add(Dict("text", line.Text, "words", words));
|
|
4318
|
+
if (joined.Length > 0) joined.Append('\n');
|
|
4319
|
+
joined.Append(line.Text);
|
|
4320
|
+
}
|
|
4321
|
+
return Dict("lines", lines, "text", joined.ToString(), "language", _ocrLanguage,
|
|
4322
|
+
"region", new int[] { x, y, w, h });
|
|
4323
|
+
}
|
|
4324
|
+
|
|
3648
4325
|
static Dictionary<string, object> Capture(Dictionary<string, object> a)
|
|
3649
4326
|
{
|
|
3650
4327
|
ValidateKickoffCapture(a); // before even the pixel-stability sampler
|
|
@@ -3693,6 +4370,7 @@ namespace DshComputerUse.Worker
|
|
|
3693
4370
|
IntPtr old = Native.SelectObject(mem, dib);
|
|
3694
4371
|
// SRCCOPY | CAPTUREBLT so layered windows are included
|
|
3695
4372
|
Native.BitBlt(mem, 0, 0, w, h, screen, x, y, 0x00CC0020 | 0x40000000);
|
|
4373
|
+
long capturedAt = NowMs();
|
|
3696
4374
|
bool cursorDrawn = false;
|
|
3697
4375
|
int cursorFlags = 0, cursorAtX = 0, cursorAtY = 0;
|
|
3698
4376
|
if (withCursor)
|
|
@@ -3755,13 +4433,16 @@ namespace DshComputerUse.Worker
|
|
|
3755
4433
|
ep.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, q);
|
|
3756
4434
|
outBmp.Save(ms, codec, ep);
|
|
3757
4435
|
}
|
|
4436
|
+
int imageWidth = outBmp.Width, imageHeight = outBmp.Height;
|
|
3758
4437
|
outBmp.Dispose();
|
|
3759
4438
|
POINT cp2; Native.GetCursorPos(out cp2);
|
|
3760
4439
|
Dictionary<string, object> res = Dict(
|
|
3761
4440
|
"image", Convert.ToBase64String(ms.ToArray()),
|
|
3762
4441
|
"mime", mime,
|
|
3763
4442
|
"region", Dict("x", x, "y", y, "width", w, "height", h),
|
|
3764
|
-
"scaledTo",
|
|
4443
|
+
"scaledTo", imageWidth,
|
|
4444
|
+
"imageSize", new int[] { imageWidth, imageHeight },
|
|
4445
|
+
"capturedAt", capturedAt,
|
|
3765
4446
|
"cursor", PtDict(cp2.X, cp2.Y),
|
|
3766
4447
|
"ts", NowMs());
|
|
3767
4448
|
if (withCursor)
|
|
@@ -3773,7 +4454,8 @@ namespace DshComputerUse.Worker
|
|
|
3773
4454
|
res["cursorDrawn"] = cursorDrawn;
|
|
3774
4455
|
res["cursorFlags"] = cursorFlags;
|
|
3775
4456
|
res["cursorScreenPos"] = PtDict(cursorAtX, cursorAtY);
|
|
3776
|
-
res["cursorInImage"] =
|
|
4457
|
+
res["cursorInImage"] = Dict("x", (cursorAtX - x) * (double)imageWidth / w,
|
|
4458
|
+
"y", (cursorAtY - y) * (double)imageHeight / h);
|
|
3777
4459
|
res["cursorNote"] = cursorDrawn
|
|
3778
4460
|
? "pointer burned into the image at cursorInImage"
|
|
3779
4461
|
: ((cursorFlags & 1) == 0
|
|
@@ -3919,6 +4601,46 @@ namespace DshComputerUse.Worker
|
|
|
3919
4601
|
SendInputChecked(new INPUT[] { inp });
|
|
3920
4602
|
}
|
|
3921
4603
|
|
|
4604
|
+
internal static string StandardCursorShape(IntPtr cursor, Func<int, IntPtr> load)
|
|
4605
|
+
{
|
|
4606
|
+
if (cursor == IntPtr.Zero) return "unknown";
|
|
4607
|
+
int[] ids = { 32644, 32645, 32643, 32642, 32646, 32512, 32513, 32649, 32514, 32515, 32648, 32650, 32651 };
|
|
4608
|
+
string[] names = { "resize-ew", "resize-ns", "resize-nesw", "resize-nwse", "move", "arrow", "ibeam", "hand", "wait", "cross", "not-allowed", "app-starting", "help" };
|
|
4609
|
+
for (int i = 0; i < ids.Length; i++) if (load(ids[i]) == cursor) return names[i];
|
|
4610
|
+
return "unknown"; // Custom/browser-drawn cursors do not reveal a CSS value.
|
|
4611
|
+
}
|
|
4612
|
+
|
|
4613
|
+
static bool CursorMatches(string expected, string actual, bool visible)
|
|
4614
|
+
{
|
|
4615
|
+
if (String.IsNullOrEmpty(expected)) return true;
|
|
4616
|
+
if (!visible || actual == "unknown") return false;
|
|
4617
|
+
return expected == "resize" ? actual.StartsWith("resize-", StringComparison.Ordinal) : expected == actual;
|
|
4618
|
+
}
|
|
4619
|
+
|
|
4620
|
+
static Dictionary<string, object> CursorEvidence(bool available, bool visible, IntPtr cursor, Func<int, IntPtr> load)
|
|
4621
|
+
{
|
|
4622
|
+
visible = available && visible;
|
|
4623
|
+
string shape = visible ? StandardCursorShape(cursor, load) : "unknown";
|
|
4624
|
+
string classification = !available ? "unavailable" : !visible ? "not-visible" : shape == "unknown" ? "unrecognized" : "standard";
|
|
4625
|
+
var result = Dict("available", available, "visible", visible, "shape", shape,
|
|
4626
|
+
"classification", classification, "source", "win32-standard-cursor");
|
|
4627
|
+
if (classification == "unrecognized")
|
|
4628
|
+
result["note"] = "Visible cursor handle does not match a supported Windows standard cursor. Browser col-resize/row-resize and custom cursors may be unrecognized; this does not prove a hit or a miss on a draggable handle.";
|
|
4629
|
+
return result;
|
|
4630
|
+
}
|
|
4631
|
+
|
|
4632
|
+
static Dictionary<string, object> CursorSnapshot()
|
|
4633
|
+
{
|
|
4634
|
+
CURSORINFO ci = new CURSORINFO();
|
|
4635
|
+
ci.cbSize = (uint)Marshal.SizeOf(typeof(CURSORINFO));
|
|
4636
|
+
bool available = Native.GetCursorInfo(ref ci);
|
|
4637
|
+
bool visible = available && (ci.flags & Native.CURSOR_SHOWING) != 0;
|
|
4638
|
+
var result = CursorEvidence(available, visible, ci.hCursor,
|
|
4639
|
+
delegate(int id) { return Native.LoadCursor(IntPtr.Zero, new IntPtr(id)); });
|
|
4640
|
+
result["screenPos"] = available ? (object)PtDict(ci.ptScreenPos.X, ci.ptScreenPos.Y) : null;
|
|
4641
|
+
return result;
|
|
4642
|
+
}
|
|
4643
|
+
|
|
3922
4644
|
static Dictionary<string, object> Move(Dictionary<string, object> a)
|
|
3923
4645
|
{
|
|
3924
4646
|
int x = GetI(a, "x", int.MinValue), y = GetI(a, "y", int.MinValue);
|
|
@@ -3940,15 +4662,80 @@ namespace DshComputerUse.Worker
|
|
|
3940
4662
|
Native.GetCursorPos(out p);
|
|
3941
4663
|
}
|
|
3942
4664
|
CheckHumanInput();
|
|
3943
|
-
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections);
|
|
4665
|
+
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections, "cursor", CursorSnapshot());
|
|
4666
|
+
}
|
|
4667
|
+
|
|
4668
|
+
static Rectangle ClickSampleRegion(int x, int y, Rectangle desktop)
|
|
4669
|
+
{
|
|
4670
|
+
int width = Math.Min(340, desktop.Width), height = Math.Min(340, desktop.Height);
|
|
4671
|
+
if (width <= 0 || height <= 0) return Rectangle.Empty;
|
|
4672
|
+
int left = (int)Math.Min(Math.Max((long)x - width / 2, desktop.Left), (long)desktop.Right - width);
|
|
4673
|
+
int top = (int)Math.Min(Math.Max((long)y - height / 2, desktop.Top), (long)desktop.Bottom - height);
|
|
4674
|
+
return new Rectangle(left, top, width, height);
|
|
4675
|
+
}
|
|
4676
|
+
|
|
4677
|
+
static int VerificationNumber(Dictionary<string, object> rectangle, string key)
|
|
4678
|
+
{
|
|
4679
|
+
object raw;
|
|
4680
|
+
if (!rectangle.TryGetValue(key, out raw) || !(raw is int || raw is long || raw is double || raw is float || raw is decimal))
|
|
4681
|
+
throw new ArgumentException("missing integer " + key);
|
|
4682
|
+
double n = Convert.ToDouble(raw);
|
|
4683
|
+
if (Double.IsNaN(n) || Double.IsInfinity(n) || n != Math.Truncate(n) || n <= Int32.MinValue || n > Int32.MaxValue)
|
|
4684
|
+
throw new ArgumentException("invalid integer " + key);
|
|
4685
|
+
return (int)n;
|
|
4686
|
+
}
|
|
4687
|
+
static Rectangle VerificationSampleRegion(Dictionary<string, object> a, int x, int y, Rectangle desktop)
|
|
4688
|
+
{
|
|
4689
|
+
if (!a.ContainsKey("verificationRegion")) return ClickSampleRegion(x, y, desktop);
|
|
4690
|
+
try {
|
|
4691
|
+
var box = a["verificationRegion"] as Dictionary<string, object>;
|
|
4692
|
+
if (box == null) throw new ArgumentException("expected rectangle object");
|
|
4693
|
+
int rx = VerificationNumber(box, "x"), ry = VerificationNumber(box, "y");
|
|
4694
|
+
int rw = VerificationNumber(box, "width"), rh = VerificationNumber(box, "height");
|
|
4695
|
+
if (rw <= 0 || rh <= 0 || (long)rw * rh > 16777216 ||
|
|
4696
|
+
(long)rx + rw > Int32.MaxValue || (long)ry + rh > Int32.MaxValue)
|
|
4697
|
+
throw new ArgumentException("positive size, 16M pixel maximum, non-overflowing edges required");
|
|
4698
|
+
Rectangle requested = new Rectangle(rx, ry, rw, rh);
|
|
4699
|
+
Rectangle visible = Rectangle.Intersect(requested, desktop);
|
|
4700
|
+
if (visible.Width <= 0 || visible.Height <= 0) throw new ArgumentException("region is outside the virtual desktop");
|
|
4701
|
+
return visible;
|
|
4702
|
+
} catch (Exception error) {
|
|
4703
|
+
var refusal = new ArgumentException("INVALID_VERIFICATION_REGION: " + error.Message + "; no input was dispatched.");
|
|
4704
|
+
refusal.Data["code"] = "INVALID_VERIFICATION_REGION"; refusal.Data["outcome"] = "not-dispatched";
|
|
4705
|
+
throw refusal;
|
|
4706
|
+
}
|
|
4707
|
+
}
|
|
4708
|
+
static void AddPixelVerification(Dictionary<string, object> result, Dictionary<string, object> args,
|
|
4709
|
+
Rectangle region, byte[] before, byte[] after)
|
|
4710
|
+
{
|
|
4711
|
+
result["effectVerified"] = false;
|
|
4712
|
+
result["verificationAvailable"] = before != null && after != null && before.Length > 0 && before.Length == after.Length;
|
|
4713
|
+
result["verificationSource"] = args.ContainsKey("verificationRegion") ? "requested-region" : "click-neighborhood";
|
|
4714
|
+
result["verificationRegion"] = Dict("x", region.X, "y", region.Y, "width", region.Width, "height", region.Height);
|
|
4715
|
+
if (args.ContainsKey("verificationRegion")) result["requestedVerificationRegion"] = args["verificationRegion"];
|
|
4716
|
+
if (!(bool)result["verificationAvailable"]) {
|
|
4717
|
+
result["verificationNote"] = "Pixel samples unavailable; input completion does not establish the intended effect.";
|
|
4718
|
+
return;
|
|
4719
|
+
}
|
|
4720
|
+
double changed = Math.Round(SigDiffPct(before, after, 8), 2);
|
|
4721
|
+
result["changedPct"] = changed;
|
|
4722
|
+
result["verificationNote"] = "Sampled pixel difference only, not semantic verification. Inspect the expected result before replaying input.";
|
|
4723
|
+
if (changed < 0.15 && !args.ContainsKey("verify"))
|
|
4724
|
+
result["warning"] = "No sampled change in the " + (args.ContainsKey("verificationRegion") ? "requested witness region" : "click neighborhood") +
|
|
4725
|
+
". This is expected for an already-focused or already-satisfied control; an effect may also be delayed or elsewhere. " +
|
|
4726
|
+
"Check the intended state and focus/receiver before retrying.";
|
|
3944
4727
|
}
|
|
3945
4728
|
|
|
3946
4729
|
static Dictionary<string, object> Click(Dictionary<string, object> a)
|
|
3947
4730
|
{
|
|
4731
|
+
Rectangle sampleRegion = VerificationSampleRegion(a, GetI(a, "x", 0), GetI(a, "y", 0), new Rectangle(
|
|
4732
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
4733
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
3948
4734
|
string btn = GetS(a, "button", "left");
|
|
3949
4735
|
int clicks = Math.Max(1, Math.Min(3, GetI(a, "clicks", 1)));
|
|
3950
4736
|
int gap = GetI(a, "intervalMs", 60);
|
|
3951
|
-
|
|
4737
|
+
bool dryRun = GetB(a, "dryRun", false);
|
|
4738
|
+
if (!dryRun) AgentActing(a, "click");
|
|
3952
4739
|
CheckHumanInput();
|
|
3953
4740
|
// If the caller NAMED the window it wants to click in, put that window in front
|
|
3954
4741
|
// first: a click on a covered pixel is delivered to whatever is on top of it (the
|
|
@@ -3959,7 +4746,6 @@ namespace DshComputerUse.Worker
|
|
|
3959
4746
|
// would change the thing it measures. That bug hid a real occlusion for a whole test
|
|
3960
4747
|
// round (2026-09-13): every dry run reported "ok" because it had just activated the
|
|
3961
4748
|
// window it was asked about.
|
|
3962
|
-
bool dryRun = GetB(a, "dryRun", false);
|
|
3963
4749
|
// A caller-supplied window is re-asserted (raised, ALT tap, titlebar click) on the
|
|
3964
4750
|
// next line, so it has to be checked BEFORE that: the pixel check further down comes
|
|
3965
4751
|
// far too late to protect a window the worker was told to bring forward.
|
|
@@ -4074,41 +4860,33 @@ namespace DshComputerUse.Worker
|
|
|
4074
4860
|
// The window under the click point becomes the sticky target for later typing.
|
|
4075
4861
|
try { SetSticky(under); } catch { }
|
|
4076
4862
|
}
|
|
4077
|
-
//
|
|
4078
|
-
//
|
|
4079
|
-
// missed pixel, wrong element) — and "no change" is exactly the silent failure that
|
|
4080
|
-
// used to be indistinguishable from success. Region sampling is ~1700 pixels, cheaper
|
|
4081
|
-
// than a screenshot by three orders of magnitude.
|
|
4082
|
-
int vsW = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
4083
|
-
int vsH = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
4863
|
+
// A cheap pixel witness, not a business-success condition. The caller may
|
|
4864
|
+
// identify an effect elsewhere (e.g. a player bar), independently of the click.
|
|
4084
4865
|
bool verify = (tx != int.MinValue) && GetB(a, "verify", true);
|
|
4085
|
-
int
|
|
4086
|
-
int bx = Math.Min(Math.Max(tx - half, margin), Math.Max(margin, vsW - margin - 2 * half));
|
|
4087
|
-
int by = Math.Min(Math.Max(ty - half, margin), Math.Max(margin, vsH - margin - 2 * half));
|
|
4088
|
-
int bw = Math.Min(2 * half, Math.Max(16, vsW - 2 * margin));
|
|
4089
|
-
int bh = Math.Min(2 * half, Math.Max(16, vsH - 2 * margin));
|
|
4866
|
+
int bx = sampleRegion.X, by = sampleRegion.Y, bw = sampleRegion.Width, bh = sampleRegion.Height;
|
|
4090
4867
|
byte[] before = verify ? FrameSignatureRect(bx, by, bw, bh, 8) : null;
|
|
4091
4868
|
|
|
4092
4869
|
// Re-ask at the moment of injection: the pointer has travelled since the precondition
|
|
4093
4870
|
// above, and the only promise that matters is about the pixel the button is really
|
|
4094
4871
|
// pressed on. Cheap (one WindowFromPoint) and it closes the window between check and use.
|
|
4095
4872
|
if (tx != int.MinValue) HostGuardPoint(tx, ty, "click");
|
|
4873
|
+
if (under == IntPtr.Zero) under = InputPointReceiver();
|
|
4096
4874
|
|
|
4097
4875
|
uint down = Native.MOUSEEVENTF_LEFTDOWN, up = Native.MOUSEEVENTF_LEFTUP;
|
|
4098
4876
|
if (btn == "right") { down = Native.MOUSEEVENTF_RIGHTDOWN; up = Native.MOUSEEVENTF_RIGHTUP; }
|
|
4099
4877
|
else if (btn == "middle") { down = Native.MOUSEEVENTF_MIDDLEDOWN; up = Native.MOUSEEVENTF_MIDDLEUP; }
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4878
|
+
using (GuardReceiver(under, InputWindowPid(under), false)) {
|
|
4879
|
+
for (int i = 0; i < clicks; i++) {
|
|
4880
|
+
SendMouse(down, 0, 0, 0); Thread.Sleep(30); SendMouse(up, 0, 0, 0);
|
|
4881
|
+
if (i < clicks - 1) Thread.Sleep(gap);
|
|
4882
|
+
}
|
|
4104
4883
|
}
|
|
4105
4884
|
EndActuationForReadback();
|
|
4106
|
-
|
|
4885
|
+
byte[] after = null;
|
|
4107
4886
|
if (verify && before != null)
|
|
4108
4887
|
{
|
|
4109
4888
|
Thread.Sleep(GetI(a, "verifyDelayMs", 240));
|
|
4110
|
-
|
|
4111
|
-
if (after != null) changedPct = Math.Round(SigDiffPct(before, after, 8), 2);
|
|
4889
|
+
after = FrameSignatureRect(bx, by, bw, bh, 8);
|
|
4112
4890
|
}
|
|
4113
4891
|
POINT fp; Native.GetCursorPos(out fp);
|
|
4114
4892
|
Dictionary<string, object> clickFocus = FocusReport(a, false);
|
|
@@ -4118,16 +4896,58 @@ namespace DshComputerUse.Worker
|
|
|
4118
4896
|
if (coveredBy != null) clickOut["coveredBy"] = coveredBy;
|
|
4119
4897
|
if (under != IntPtr.Zero)
|
|
4120
4898
|
clickOut["receiver"] = Short(WinTitleOf(under), 40) + " (pid " + PidOf(under) + ")";
|
|
4121
|
-
if (
|
|
4122
|
-
{
|
|
4123
|
-
clickOut["changedPct"] = changedPct;
|
|
4124
|
-
if (changedPct < 0.15 && !a.ContainsKey("verify"))
|
|
4125
|
-
clickOut["warning"] = "nothing changed near the click point within ~240ms: the click may not have reached " +
|
|
4126
|
-
"its target (dead UI, disabled control, or a wrong pixel). Probe the point before retrying.";
|
|
4127
|
-
}
|
|
4899
|
+
if (verify) AddPixelVerification(clickOut, a, sampleRegion, before, after);
|
|
4128
4900
|
return clickOut;
|
|
4129
4901
|
}
|
|
4130
4902
|
|
|
4903
|
+
static Exception DragPrecondition(string code, string detail)
|
|
4904
|
+
{
|
|
4905
|
+
var error = InputInterrupted(code, detail + " No drag button was pressed; earlier focus or pointer movement may have occurred.");
|
|
4906
|
+
error.Data["dragStarted"] = false;
|
|
4907
|
+
return error;
|
|
4908
|
+
}
|
|
4909
|
+
|
|
4910
|
+
static Dictionary<string, object> CheckDragCursor(string expected, Dictionary<string, object> actual)
|
|
4911
|
+
{
|
|
4912
|
+
expected = (expected ?? "").Trim().ToLowerInvariant();
|
|
4913
|
+
string shape = (string)actual["shape"];
|
|
4914
|
+
string classification = (string)actual["classification"];
|
|
4915
|
+
string status = expected.Length == 0 ? "not-requested" : CursorMatches(expected, shape, (bool)actual["visible"])
|
|
4916
|
+
? "matched" : classification == "standard" ? "mismatched" : "unverifiable";
|
|
4917
|
+
var check = Dict("status", status, "expected", expected.Length == 0 ? null : (object)expected, "observed", shape);
|
|
4918
|
+
if (status == "mismatched" || status == "unverifiable") {
|
|
4919
|
+
string detail = status == "mismatched"
|
|
4920
|
+
? "Expected " + expected + ", observed standard cursor " + shape + ". Re-observe the intended handle before retrying."
|
|
4921
|
+
: "Cannot verify expected cursor " + expected + ": classification=" + classification + ". An unknown/unavailable cursor is not proof of an incorrect target. Browser col-resize/row-resize may use nonstandard cursors. If fresh independent geometry and hit-testing confirm the handle, the optional expectCursor may be omitted; verify the application result afterward.";
|
|
4922
|
+
// Keep the existing error code for callers; cursorCheck distinguishes a known
|
|
4923
|
+
// shape disagreement from missing evidence. Neither is converted to a match.
|
|
4924
|
+
var error = DragPrecondition("CURSOR_MISMATCH", detail);
|
|
4925
|
+
error.Data["expectedCursor"] = expected;
|
|
4926
|
+
error.Data["actualCursor"] = actual;
|
|
4927
|
+
error.Data["cursorCheck"] = check;
|
|
4928
|
+
throw error;
|
|
4929
|
+
}
|
|
4930
|
+
return check;
|
|
4931
|
+
}
|
|
4932
|
+
|
|
4933
|
+
static IntPtr CheckDragStart(Dictionary<string, object> a, int x, int y)
|
|
4934
|
+
{
|
|
4935
|
+
HostGuardPoint(x, y, "drag");
|
|
4936
|
+
POINT point; point.X = x; point.Y = y;
|
|
4937
|
+
IntPtr under = TopLevel(Native.WindowFromPoint(point));
|
|
4938
|
+
IntPtr want = a.ContainsKey("expectHwnd") ? new IntPtr(GetI(a, "expectHwnd", 0)) : _stickyHwnd;
|
|
4939
|
+
if (under == IntPtr.Zero) throw DragPrecondition("DRAG_START_UNREACHABLE", "No window owns the drag start pixel.");
|
|
4940
|
+
if (want != IntPtr.Zero) {
|
|
4941
|
+
if (!Native.IsWindow(want) || Native.IsIconic(want) || IsOffScreen(want))
|
|
4942
|
+
throw DragPrecondition("DRAG_TARGET_UNAVAILABLE", "The expected drag window is gone, minimized or off-screen.");
|
|
4943
|
+
if (under != want && PidOf(under) != PidOf(want))
|
|
4944
|
+
throw DragPrecondition("DRAG_START_COVERED", "The start pixel belongs to hwnd=" + under.ToInt64() + ", not expected hwnd=" + want.ToInt64() + ". Re-observe the target.");
|
|
4945
|
+
}
|
|
4946
|
+
if (ProcessElevated(PidOf(under)) && !SelfElevated())
|
|
4947
|
+
throw DragPrecondition("DRAG_START_ELEVATED", "The drag receiver is elevated and this worker cannot inject into it.");
|
|
4948
|
+
return under;
|
|
4949
|
+
}
|
|
4950
|
+
|
|
4131
4951
|
static Dictionary<string, object> Drag(Dictionary<string, object> a)
|
|
4132
4952
|
{
|
|
4133
4953
|
int fx = GetI(a, "fromX", int.MinValue), fy = GetI(a, "fromY", int.MinValue);
|
|
@@ -4142,9 +4962,22 @@ namespace DshComputerUse.Worker
|
|
|
4142
4962
|
int durMs = Math.Max(50, GetI(a, "durationMs", 350));
|
|
4143
4963
|
AgentActing(a, "drag");
|
|
4144
4964
|
CheckHumanInput();
|
|
4965
|
+
if (a.ContainsKey("expectHwnd")) {
|
|
4966
|
+
HostGuardNamed(a, "drag");
|
|
4967
|
+
FocusReport(a, true);
|
|
4968
|
+
}
|
|
4969
|
+
CheckDragStart(a, fx, fy);
|
|
4145
4970
|
AnimatedMove(fx, fy, GetI(a, "moveDurationMs", 250));
|
|
4146
4971
|
Thread.Sleep(80);
|
|
4147
|
-
|
|
4972
|
+
CheckHumanInput();
|
|
4973
|
+
POINT startPoint;
|
|
4974
|
+
if (!Native.GetCursorPos(out startPoint) || Dist(startPoint, fx, fy) > 2)
|
|
4975
|
+
throw DragPrecondition("DRAG_START_MISSED", "The pointer did not reach the measured drag start within two screen pixels.");
|
|
4976
|
+
IntPtr startWindow = CheckDragStart(a, startPoint.X, startPoint.Y);
|
|
4977
|
+
Dictionary<string, object> startCursor = CursorSnapshot();
|
|
4978
|
+
Dictionary<string, object> cursorCheck = CheckDragCursor(GetS(a, "expectCursor", ""), startCursor);
|
|
4979
|
+
using (GuardReceiver(startWindow, InputWindowPid(startWindow), false))
|
|
4980
|
+
SendMouse(Native.MOUSEEVENTF_LEFTDOWN, 0, 0, 0);
|
|
4148
4981
|
Thread.Sleep(80);
|
|
4149
4982
|
POINT probe0; Native.GetCursorPos(out probe0);
|
|
4150
4983
|
if (Dist(probe0, fx, fy) > 24)
|
|
@@ -4173,7 +5006,9 @@ namespace DshComputerUse.Worker
|
|
|
4173
5006
|
SendMouse(Native.MOUSEEVENTF_LEFTUP, 0, 0, 0);
|
|
4174
5007
|
POINT p; Native.GetCursorPos(out p);
|
|
4175
5008
|
CheckHumanInput();
|
|
4176
|
-
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections
|
|
5009
|
+
return Dict("x", p.X, "y", p.Y, "positionCorrections", _positionCorrections,
|
|
5010
|
+
"dragStarted", true, "startCursor", startCursor, "cursorCheck", cursorCheck, "startHwnd", startWindow.ToInt64(),
|
|
5011
|
+
"effectVerified", false, "note", "Input sequence completed. Cursor/window checks do not prove that a handle moved; verify the intended UI change.");
|
|
4177
5012
|
}
|
|
4178
5013
|
|
|
4179
5014
|
static Dictionary<string, object> Scroll(Dictionary<string, object> a)
|
|
@@ -4188,10 +5023,14 @@ namespace DshComputerUse.Worker
|
|
|
4188
5023
|
else HostGuardCursor("scroll");
|
|
4189
5024
|
AgentActing(a, "scroll");
|
|
4190
5025
|
if (a.ContainsKey("x") && a.ContainsKey("y")) Move(a);
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
5026
|
+
IntPtr scrollTarget = InputPointReceiver();
|
|
5027
|
+
if (_stickyHwnd != IntPtr.Zero && scrollTarget != _stickyHwnd && InputWindowPid(scrollTarget) != InputWindowPid(_stickyHwnd))
|
|
5028
|
+
throw InputInterrupted("INPUT_TARGET_CHANGED", "Scroll receiver differs from the observed target. No wheel input was sent.");
|
|
5029
|
+
using (GuardReceiver(scrollTarget, InputWindowPid(scrollTarget), false)) {
|
|
5030
|
+
for (int i = 0; i < lines; i++) {
|
|
5031
|
+
SendMouse(flag, 0, 0, unchecked((uint)(sign * (int)Native.WHEEL_DELTA)));
|
|
5032
|
+
Thread.Sleep(25);
|
|
5033
|
+
}
|
|
4195
5034
|
}
|
|
4196
5035
|
POINT p; Native.GetCursorPos(out p);
|
|
4197
5036
|
return PtDict(p.X, p.Y);
|
|
@@ -4235,6 +5074,14 @@ namespace DshComputerUse.Worker
|
|
|
4235
5074
|
|
|
4236
5075
|
static Dictionary<string, object> Indicator(Dictionary<string, object> a)
|
|
4237
5076
|
{
|
|
5077
|
+
if (a.ContainsKey("hold"))
|
|
5078
|
+
{
|
|
5079
|
+
SceneControl.SetHolding(GetB(a, "hold", false));
|
|
5080
|
+
if (SceneControl.Holding) Glow.Reveal();
|
|
5081
|
+
if (Panic.CycleLit && !Panic.Exited) Glow.Thinking(0);
|
|
5082
|
+
return Dict("holding", Glow.Holding, "cycleActive", Panic.CycleLit && !Panic.Exited,
|
|
5083
|
+
"indicator", Glow.Holding ? "holding" : "free");
|
|
5084
|
+
}
|
|
4238
5085
|
bool on = GetB(a, "on", true);
|
|
4239
5086
|
bool soft = GetB(a, "soft", false);
|
|
4240
5087
|
if (on) { Glow.Enabled = true; Glow.Touch(GetI(a, "ms", 30000)); }
|
|
@@ -4318,20 +5165,38 @@ namespace DshComputerUse.Worker
|
|
|
4318
5165
|
{
|
|
4319
5166
|
int x = GetI(a, "x", int.MinValue), y = GetI(a, "y", int.MinValue);
|
|
4320
5167
|
if (x == int.MinValue || y == int.MinValue) throw new Exception("x,y required");
|
|
5168
|
+
Rectangle witness = VerificationSampleRegion(a, x, y, new Rectangle(
|
|
5169
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
5170
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
4321
5171
|
HostGuardPoint(x, y, "shiftClick");
|
|
5172
|
+
POINT targetPoint; targetPoint.X = x; targetPoint.Y = y;
|
|
5173
|
+
IntPtr targetWindow = TopLevel(Native.WindowFromPoint(targetPoint));
|
|
5174
|
+
if (targetWindow == IntPtr.Zero) throw InputInterrupted("INPUT_TARGET_CHANGED", "Shift-click target is unavailable; no input sent.");
|
|
5175
|
+
uint targetPid = InputWindowPid(targetWindow);
|
|
4322
5176
|
int clicks = Math.Max(1, Math.Min(3, GetI(a, "clicks", 1)));
|
|
4323
5177
|
AgentActing(a, "shiftClick");
|
|
4324
5178
|
CheckHumanInput();
|
|
4325
5179
|
AnimatedMove(x, y, Math.Max(60, GetI(a, "moveDurationMs", 200)));
|
|
4326
5180
|
Thread.Sleep(GetI(a, "settleMs", 90));
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
5181
|
+
byte[] before = a.ContainsKey("verificationRegion") ? FrameSignatureRect(witness.X, witness.Y, witness.Width, witness.Height, 8) : null;
|
|
5182
|
+
using (GuardReceiver(targetWindow, targetPid, false)) {
|
|
5183
|
+
SendKey(0x10, false); // VK_SHIFT down
|
|
5184
|
+
Thread.Sleep(40);
|
|
5185
|
+
ClickTimes(clicks);
|
|
5186
|
+
Thread.Sleep(40);
|
|
5187
|
+
SendKey(0x10, true); // VK_SHIFT up
|
|
5188
|
+
}
|
|
4332
5189
|
Thread.Sleep(GetI(a, "settleMs", 90));
|
|
4333
5190
|
CheckHumanInput();
|
|
4334
|
-
|
|
5191
|
+
var result = Dict("x", x, "y", y, "clicks", clicks);
|
|
5192
|
+
if (a.ContainsKey("verificationRegion")) {
|
|
5193
|
+
EndActuationForReadback();
|
|
5194
|
+
Thread.Sleep(240);
|
|
5195
|
+
byte[] after = FrameSignatureRect(witness.X, witness.Y, witness.Width, witness.Height, 8);
|
|
5196
|
+
CheckHumanInput();
|
|
5197
|
+
AddPixelVerification(result, a, witness, before, after);
|
|
5198
|
+
}
|
|
5199
|
+
return result;
|
|
4335
5200
|
}
|
|
4336
5201
|
|
|
4337
5202
|
static void ClickTimes(int n)
|
|
@@ -4420,7 +5285,11 @@ namespace DshComputerUse.Worker
|
|
|
4420
5285
|
IntPtr dib = Native.CreateDIBSection(screen, ref bmi, 0, out bits, IntPtr.Zero, 0);
|
|
4421
5286
|
if (dib == IntPtr.Zero) { Native.DeleteDC(mem); Native.ReleaseDC(IntPtr.Zero, screen); return null; }
|
|
4422
5287
|
IntPtr old = Native.SelectObject(mem, dib);
|
|
4423
|
-
Native.BitBlt(mem, 0, 0, rw, rh, screen, rx, ry, 0x00CC0020 | 0x40000000)
|
|
5288
|
+
if (!Native.BitBlt(mem, 0, 0, rw, rh, screen, rx, ry, 0x00CC0020 | 0x40000000)) {
|
|
5289
|
+
Native.SelectObject(mem, old);
|
|
5290
|
+
Native.DeleteObject(dib); Native.DeleteDC(mem); Native.ReleaseDC(IntPtr.Zero, screen);
|
|
5291
|
+
return null;
|
|
5292
|
+
}
|
|
4424
5293
|
int n = ((rw - 1) / step + 1) * ((rh - 1) / step + 1);
|
|
4425
5294
|
byte[] sig = new byte[n];
|
|
4426
5295
|
int k = 0;
|
|
@@ -4534,11 +5403,16 @@ namespace DshComputerUse.Worker
|
|
|
4534
5403
|
|
|
4535
5404
|
// Explicit window scope excludes unrelated desktop animation without loosening the
|
|
4536
5405
|
// change threshold. A moved/minimized/replaced window invalidates the observation.
|
|
4537
|
-
static Func<byte[]> WaitSampler(Dictionary<string, object> a, int step, out object scope
|
|
5406
|
+
static Func<byte[]> WaitSampler(Dictionary<string, object> a, int step, out object scope,
|
|
5407
|
+
out int rx, out int ry, out int rw, out int rh)
|
|
4538
5408
|
{
|
|
4539
5409
|
if (!a.ContainsKey("hwnd") || GetI(a, "hwnd", 0) == 0)
|
|
4540
5410
|
{
|
|
4541
|
-
|
|
5411
|
+
rx = Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN);
|
|
5412
|
+
ry = Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN);
|
|
5413
|
+
rw = Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN);
|
|
5414
|
+
rh = Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN);
|
|
5415
|
+
scope = Dict("kind", "virtual-desktop", "region", Dict("x", rx, "y", ry, "width", rw, "height", rh));
|
|
4542
5416
|
return delegate { return FrameSignature(step); };
|
|
4543
5417
|
}
|
|
4544
5418
|
IntPtr h = new IntPtr(GetI(a, "hwnd", 0));
|
|
@@ -4550,7 +5424,8 @@ namespace DshComputerUse.Worker
|
|
|
4550
5424
|
Rectangle visible = Rectangle.Intersect(virtualRect, Rectangle.FromLTRB(original.Left, original.Top, original.Right, original.Bottom));
|
|
4551
5425
|
if (visible.Width <= 16 || visible.Height <= 16)
|
|
4552
5426
|
throw new Exception("WAIT_TARGET_UNAVAILABLE: window has no usable visible screen region");
|
|
4553
|
-
|
|
5427
|
+
rx = visible.X; ry = visible.Y; rw = visible.Width; rh = visible.Height;
|
|
5428
|
+
scope = Dict("kind", "window", "hwnd", h.ToInt64(), "region", Dict("x", rx, "y", ry, "width", rw, "height", rh));
|
|
4554
5429
|
return delegate {
|
|
4555
5430
|
RECT current;
|
|
4556
5431
|
if (!Native.IsWindow(h) || Native.IsIconic(h) || PidOf(h) != pid || !Native.GetWindowRect(h, out current) ||
|
|
@@ -4560,6 +5435,29 @@ namespace DshComputerUse.Worker
|
|
|
4560
5435
|
};
|
|
4561
5436
|
}
|
|
4562
5437
|
|
|
5438
|
+
/// Bounding box (absolute screen px) of every signature cell that differs beyond the
|
|
5439
|
+
/// threshold — "where did it change", not just "did it" (test report 2026-11-04: a
|
|
5440
|
+
/// 0.344% diff cannot distinguish a sent message from a blinking caret without this).
|
|
5441
|
+
static int[] SigDiffBounds(byte[] a, byte[] b, int threshold, int cols, int step, int rx, int ry)
|
|
5442
|
+
{
|
|
5443
|
+
if (a == null || b == null || a.Length != b.Length || cols <= 0) return null;
|
|
5444
|
+
int minX = -1, minY = -1, maxX = -1, maxY = -1;
|
|
5445
|
+
for (int i = 0; i < a.Length; i++)
|
|
5446
|
+
{
|
|
5447
|
+
int d = a[i] - b[i];
|
|
5448
|
+
if (d < 0) d = -d;
|
|
5449
|
+
if (d <= threshold) continue;
|
|
5450
|
+
int c = i % cols, r = i / cols;
|
|
5451
|
+
if (minX < 0 || c < minX) minX = c;
|
|
5452
|
+
if (c > maxX) maxX = c;
|
|
5453
|
+
if (minY < 0 || r < minY) minY = r;
|
|
5454
|
+
if (r > maxY) maxY = r;
|
|
5455
|
+
}
|
|
5456
|
+
if (minX < 0) return null;
|
|
5457
|
+
return new int[] { rx + minX * step, ry + minY * step,
|
|
5458
|
+
(maxX - minX + 1) * step, (maxY - minY + 1) * step };
|
|
5459
|
+
}
|
|
5460
|
+
|
|
4563
5461
|
static Dictionary<string, object> WaitStable(Dictionary<string, object> a)
|
|
4564
5462
|
{
|
|
4565
5463
|
int capMs = Math.Max(400, GetI(a, "timeoutMs", 8000));
|
|
@@ -4568,8 +5466,34 @@ namespace DshComputerUse.Worker
|
|
|
4568
5466
|
int step = Math.Max(4, Math.Min(64, GetI(a, "step", 8)));
|
|
4569
5467
|
double diffPct = GetD(a, "diffPct", 0.30);
|
|
4570
5468
|
int waited, estimate; double lastDiff;
|
|
4571
|
-
|
|
5469
|
+
int rx, ry, rw, rh;
|
|
5470
|
+
object scope; Func<byte[]> sample = WaitSampler(a, step, out scope, out rx, out ry, out rw, out rh);
|
|
4572
5471
|
bool ok = SettleSampleWait(capMs, stableMs, pollMs, diffPct, sample, out waited, out lastDiff, out estimate);
|
|
5472
|
+
if (!ok)
|
|
5473
|
+
{
|
|
5474
|
+
// CARET / BLINK DETECTION (test report 2026-11-04): a blinking caret is a ~1-2 Hz
|
|
5475
|
+
// periodic sub-percent change that never settles — Notepad burned the full 8 s cap
|
|
5476
|
+
// on lastDiffPct 0.388%. Probe one extra window (~2.4 s covers several blink
|
|
5477
|
+
// cycles): if EVERY consecutive diff is tiny but nonzero, the only motion is the
|
|
5478
|
+
// caret — call it stable and say exactly why.
|
|
5479
|
+
int probes = Math.Max(6, 2400 / pollMs);
|
|
5480
|
+
byte[] prev = sample();
|
|
5481
|
+
double max = 0; bool zero = false, large = false;
|
|
5482
|
+
for (int i = 0; i < probes; i++)
|
|
5483
|
+
{
|
|
5484
|
+
Thread.Sleep(pollMs);
|
|
5485
|
+
byte[] cur = sample();
|
|
5486
|
+
double d = SigDiffPct(prev, cur, 8);
|
|
5487
|
+
prev = cur;
|
|
5488
|
+
if (d == 0) zero = true;
|
|
5489
|
+
if (d > 1.0) large = true;
|
|
5490
|
+
if (d > max) max = d;
|
|
5491
|
+
}
|
|
5492
|
+
if (!large && !zero && max < 1.0)
|
|
5493
|
+
return Dict("stable", true, "waitedMs", waited, "lastDiffPct", Math.Round(max, 3),
|
|
5494
|
+
"estimateMs", estimate, "scope", scope, "caretSteady", true,
|
|
5495
|
+
"note", "the only sampled motion is a small periodic change consistent with a blinking caret (~1-2 Hz); treated as stable. lastDiffPct here is the LARGEST probe difference. This still does not verify task completion.");
|
|
5496
|
+
}
|
|
4573
5497
|
return Dict("stable", ok, "waitedMs", waited, "lastDiffPct", Math.Round(lastDiff, 3),
|
|
4574
5498
|
"estimateMs", estimate, "scope", scope,
|
|
4575
5499
|
"note", ok ? "sampled pixels settled; this does not verify task completion"
|
|
@@ -4582,10 +5506,12 @@ namespace DshComputerUse.Worker
|
|
|
4582
5506
|
int pollMs = Math.Max(100, GetI(a, "pollMs", 220));
|
|
4583
5507
|
int step = Math.Max(4, Math.Min(64, GetI(a, "step", 8)));
|
|
4584
5508
|
double diffPct = GetD(a, "diffPct", 0.30);
|
|
4585
|
-
|
|
5509
|
+
int rx, ry, rw, rh;
|
|
5510
|
+
object scope; Func<byte[]> sample = WaitSampler(a, step, out scope, out rx, out ry, out rw, out rh);
|
|
4586
5511
|
byte[] first = sample();
|
|
4587
5512
|
long t0 = NowMs();
|
|
4588
5513
|
int samples = 0;
|
|
5514
|
+
int cols = (rw - 1) / step + 1;
|
|
4589
5515
|
while (NowMs() - t0 < timeoutMs)
|
|
4590
5516
|
{
|
|
4591
5517
|
Thread.Sleep(pollMs);
|
|
@@ -4593,7 +5519,15 @@ namespace DshComputerUse.Worker
|
|
|
4593
5519
|
samples++;
|
|
4594
5520
|
double d = SigDiffPct(first, cur, 8);
|
|
4595
5521
|
if (d > diffPct)
|
|
4596
|
-
|
|
5522
|
+
{
|
|
5523
|
+
// WHERE it changed (test report 2026-11-04): 0.344% cannot tell a sent
|
|
5524
|
+
// message from a caret blink; the bounding box can — input area vs caret.
|
|
5525
|
+
int[] bounds = SigDiffBounds(first, cur, 8, cols, step, rx, ry);
|
|
5526
|
+
Dictionary<string, object> hit = Dict("changed", true, "afterMs", NowMs() - t0,
|
|
5527
|
+
"diffPct", Math.Round(d, 3), "samples", samples, "scope", scope);
|
|
5528
|
+
if (bounds != null) hit["changeBounds"] = Dict("x", bounds[0], "y", bounds[1], "width", bounds[2], "height", bounds[3]);
|
|
5529
|
+
return hit;
|
|
5530
|
+
}
|
|
4597
5531
|
}
|
|
4598
5532
|
return Dict("changed", false, "afterMs", NowMs() - t0, "samples", samples, "scope", scope,
|
|
4599
5533
|
"note", "no sampled pixel change exceeded the threshold; inspect task state before retrying a write");
|
|
@@ -4778,6 +5712,35 @@ namespace DshComputerUse.Worker
|
|
|
4778
5712
|
}
|
|
4779
5713
|
throw new Exception("unknown key: " + k);
|
|
4780
5714
|
}
|
|
5715
|
+
static string NormalizeKeyCombo(string combo)
|
|
5716
|
+
{
|
|
5717
|
+
if (String.IsNullOrWhiteSpace(combo)) throw new Exception("key combo required");
|
|
5718
|
+
var ordered = new List<string>(); var seen = new HashSet<string>(StringComparer.Ordinal);
|
|
5719
|
+
foreach (string raw in combo.Split('+')) {
|
|
5720
|
+
string key = raw.Trim().ToLowerInvariant();
|
|
5721
|
+
if (key.Length == 0) throw new Exception("key combo contains an empty token; use plus for the + key");
|
|
5722
|
+
switch (key) {
|
|
5723
|
+
case "control": key = "ctrl"; break;
|
|
5724
|
+
case "lwin": case "windows": case "meta": key = "win"; break;
|
|
5725
|
+
case "return": key = "enter"; break;
|
|
5726
|
+
case "escape": key = "esc"; break;
|
|
5727
|
+
case "bksp": key = "backspace"; break;
|
|
5728
|
+
case "del": key = "delete"; break;
|
|
5729
|
+
case "ins": key = "insert"; break;
|
|
5730
|
+
case "pageup": key = "pgup"; break;
|
|
5731
|
+
case "pagedown": key = "pgdn"; break;
|
|
5732
|
+
case "capslock": key = "caps"; break;
|
|
5733
|
+
case "printscreen": key = "prtsc"; break;
|
|
5734
|
+
}
|
|
5735
|
+
VkOf(key); // parser and native dispatch accept the same key set
|
|
5736
|
+
if (seen.Add(key)) ordered.Add(key);
|
|
5737
|
+
}
|
|
5738
|
+
var result = new List<string>();
|
|
5739
|
+
foreach (string modifier in new string[] { "ctrl", "alt", "shift", "win", "rwin" })
|
|
5740
|
+
if (seen.Contains(modifier)) { result.Add(modifier); ordered.Remove(modifier); }
|
|
5741
|
+
result.AddRange(ordered);
|
|
5742
|
+
return String.Join("+", result);
|
|
5743
|
+
}
|
|
4781
5744
|
|
|
4782
5745
|
static void SendKey(ushort vk, bool up)
|
|
4783
5746
|
{
|
|
@@ -4817,11 +5780,11 @@ namespace DshComputerUse.Worker
|
|
|
4817
5780
|
|
|
4818
5781
|
static Dictionary<string, object> Key(Dictionary<string, object> a)
|
|
4819
5782
|
{
|
|
5783
|
+
if (_receiverCheck != null) _receiverCheck(); // nested paste may not silently repair a stolen receiver
|
|
4820
5784
|
string combo = GetS(a, "combo", null);
|
|
4821
5785
|
if (combo == null) throw new Exception("combo required (e.g. 'ctrl+shift+t', 'enter')");
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
if (keys.Count == 0) throw new Exception("empty combo");
|
|
5786
|
+
combo = NormalizeKeyCombo(combo);
|
|
5787
|
+
List<string> keys = new List<string>(combo.Split('+'));
|
|
4825
5788
|
AgentActing(a, "key");
|
|
4826
5789
|
// Keyboard input goes to the FOREGROUND window — re-assert the window the agent is
|
|
4827
5790
|
// working in, and report which window actually received the keys.
|
|
@@ -4836,18 +5799,38 @@ namespace DshComputerUse.Worker
|
|
|
4836
5799
|
HostGuardForeground("key");
|
|
4837
5800
|
List<ushort> vks = new List<ushort>();
|
|
4838
5801
|
foreach (string k in keys) { ushort vk = VkOf(k); if (!vks.Contains(vk)) vks.Add(vk); }
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
5802
|
+
bool windowManagement = WindowManagementChord.IsExact(combo);
|
|
5803
|
+
using (GuardReceiver(new IntPtr(Convert.ToInt64(focus["hwnd"])), Convert.ToUInt32(focus["pid"]), true))
|
|
5804
|
+
{
|
|
5805
|
+
if (windowManagement)
|
|
5806
|
+
WindowManagementChord.Run(vks, SendKey, ReleaseKeyForCleanup,
|
|
5807
|
+
delegate(ushort vk) { return (Native.GetAsyncKeyState(vk) & 0x8000) != 0; },
|
|
5808
|
+
delegate { WaitForKeyHold(Math.Max(20, GetI(a, "holdMs", 60))); },
|
|
5809
|
+
delegate { WaitForKeyHold(0); }, InvalidateKeyboardTarget);
|
|
5810
|
+
else
|
|
5811
|
+
KeyboardChord.Run(vks, SendKey, ReleaseKeyForCleanup,
|
|
5812
|
+
delegate(ushort vk) { return (Native.GetAsyncKeyState(vk) & 0x8000) != 0; },
|
|
5813
|
+
delegate { WaitForKeyHold(Math.Max(20, GetI(a, "holdMs", 60))); });
|
|
5814
|
+
}
|
|
4842
5815
|
EndActuationForReadback();
|
|
4843
5816
|
Thread.Sleep(30);
|
|
4844
|
-
Dictionary<string, object> keyOut = Dict("combo", combo,
|
|
5817
|
+
Dictionary<string, object> keyOut = AddFocusReadback(Dict("combo", combo), a, focus);
|
|
5818
|
+
if (windowManagement)
|
|
5819
|
+
{
|
|
5820
|
+
var after = (Dictionary<string, object>)keyOut["focusAfter"];
|
|
5821
|
+
keyOut["receiverChanged"] = Convert.ToInt64(focus["hwnd"]) != Convert.ToInt64(after["hwnd"]) ||
|
|
5822
|
+
Convert.ToUInt32(focus["pid"]) != Convert.ToUInt32(after["pid"]);
|
|
5823
|
+
keyOut["reobserveRequired"] = true;
|
|
5824
|
+
keyOut["effectVerified"] = false;
|
|
5825
|
+
keyOut["hint"] = KeyboardReobserveHint;
|
|
5826
|
+
}
|
|
4845
5827
|
if (_lastRefocus != null) keyOut["refocused"] = _lastRefocus;
|
|
4846
5828
|
return keyOut;
|
|
4847
5829
|
}
|
|
4848
5830
|
|
|
4849
5831
|
static Dictionary<string, object> Type(Dictionary<string, object> a)
|
|
4850
5832
|
{
|
|
5833
|
+
if (_receiverCheck != null) _receiverCheck();
|
|
4851
5834
|
string text = GetS(a, "text", null);
|
|
4852
5835
|
if (text == null) throw new Exception("text required");
|
|
4853
5836
|
AgentActing(a, "type");
|
|
@@ -4856,6 +5839,7 @@ namespace DshComputerUse.Worker
|
|
|
4856
5839
|
HostGuardNamed(a, "type");
|
|
4857
5840
|
Dictionary<string, object> focus = FocusReport(a, true);
|
|
4858
5841
|
HostGuardForeground("type"); // same rule as `key`: the foreground window is the victim
|
|
5842
|
+
using (GuardReceiver(new IntPtr(Convert.ToInt64(focus["hwnd"])), Convert.ToUInt32(focus["pid"]), true)) {
|
|
4859
5843
|
// ...and whether whatever owns the focus can hold text at all. Typing into a control
|
|
4860
5844
|
// with no editable pattern is silently discarded, or interpreted as shortcuts.
|
|
4861
5845
|
string typeWarn = null;
|
|
@@ -4892,6 +5876,24 @@ namespace DshComputerUse.Worker
|
|
|
4892
5876
|
{
|
|
4893
5877
|
CheckHumanInput();
|
|
4894
5878
|
ClipboardLease lease = ClipboardLease.Begin(access, text);
|
|
5879
|
+
if (lease == null)
|
|
5880
|
+
{
|
|
5881
|
+
// UNPRESERVABLE CLIPBOARD, 2026-11-04: the previous clipboard holds
|
|
5882
|
+
// private/oversized content we cannot snapshot. Overwrite and paste
|
|
5883
|
+
// anyway — a lost opaque clipboard item is recoverable, while the old
|
|
5884
|
+
// refusal fed CJK text to the unicode fallback and an IME-garbled
|
|
5885
|
+
// message actually went out (WeChat, format 49161).
|
|
5886
|
+
MarkMutation();
|
|
5887
|
+
Clipboard.SetDataObject(text, true);
|
|
5888
|
+
Thread.Sleep(60);
|
|
5889
|
+
Key(Dict("combo", "ctrl+v", "holdMs", 40, "indicatorMs", 800));
|
|
5890
|
+
Dictionary<string, object> downgraded = Dict("mode", "paste", "chars", text.Length, "focus", focus,
|
|
5891
|
+
"clipboardRestore", "skipped-unpreservable-clipboard");
|
|
5892
|
+
downgraded["clipboardWarning"] = "The previous clipboard could not be preserved (private or oversized formats) and was overwritten by this paste. Restore it manually if it mattered.";
|
|
5893
|
+
if (typeWarn != null) downgraded["warning"] = typeWarn;
|
|
5894
|
+
if (_lastRefocus != null) downgraded["refocused"] = _lastRefocus;
|
|
5895
|
+
return AddFocusReadback(downgraded, a, focus);
|
|
5896
|
+
}
|
|
4895
5897
|
Exception inputError = null;
|
|
4896
5898
|
try
|
|
4897
5899
|
{
|
|
@@ -4924,7 +5926,7 @@ namespace DshComputerUse.Worker
|
|
|
4924
5926
|
pr["clipboardWarning"] = "Clipboard changed or could not be restored. Verify the target value before repeating input.";
|
|
4925
5927
|
if (typeWarn != null) pr["warning"] = typeWarn;
|
|
4926
5928
|
if (_lastRefocus != null) pr["refocused"] = _lastRefocus;
|
|
4927
|
-
return pr;
|
|
5929
|
+
return AddFocusReadback(pr, a, focus);
|
|
4928
5930
|
}
|
|
4929
5931
|
int charDelay = GetI(a, "charDelayMs", 8);
|
|
4930
5932
|
foreach (char ch in text)
|
|
@@ -4948,10 +5950,17 @@ namespace DshComputerUse.Worker
|
|
|
4948
5950
|
SendInputChecked(new INPUT[] { down, up });
|
|
4949
5951
|
Thread.Sleep(charDelay);
|
|
4950
5952
|
}
|
|
4951
|
-
Dictionary<string, object> tr = Dict("mode", "unicode", "chars", text.Length
|
|
5953
|
+
Dictionary<string, object> tr = Dict("mode", "unicode", "chars", text.Length);
|
|
4952
5954
|
if (typeWarn != null) tr["warning"] = typeWarn;
|
|
4953
5955
|
if (_lastRefocus != null) tr["refocused"] = _lastRefocus;
|
|
4954
|
-
return tr;
|
|
5956
|
+
return AddFocusReadback(tr, a, focus);
|
|
5957
|
+
}
|
|
5958
|
+
}
|
|
5959
|
+
static Dictionary<string, object> AddFocusReadback(Dictionary<string, object> result, Dictionary<string, object> args, Dictionary<string, object> before)
|
|
5960
|
+
{
|
|
5961
|
+
var after = FocusReport(args, false);
|
|
5962
|
+
result["focusBefore"] = before; result["focusAfter"] = after; result["focus"] = after;
|
|
5963
|
+
return result;
|
|
4955
5964
|
}
|
|
4956
5965
|
|
|
4957
5966
|
// ---------- clipboard ----------
|
|
@@ -4995,16 +6004,49 @@ namespace DshComputerUse.Worker
|
|
|
4995
6004
|
|
|
4996
6005
|
static Dictionary<string, object> Windows(Dictionary<string, object> a)
|
|
4997
6006
|
{
|
|
6007
|
+
uint self = (uint)System.Diagnostics.Process.GetCurrentProcess().Id;
|
|
4998
6008
|
List<object> list = new List<object>();
|
|
4999
6009
|
Native.EnumWindows(delegate(IntPtr h, IntPtr l)
|
|
5000
6010
|
{
|
|
5001
6011
|
if (!Native.IsWindowVisible(h)) return true;
|
|
5002
6012
|
StringBuilder sb = new StringBuilder(512);
|
|
5003
6013
|
if (Native.GetWindowText(h, sb, 512) == 0) return true;
|
|
6014
|
+
// Hide this worker's own click-through indicator overlay (test report 2026-11-04):
|
|
6015
|
+
// it is a full-screen topmost layer, never a target, and listing it only invites
|
|
6016
|
+
// the model to "activate" the thing that must never take focus.
|
|
6017
|
+
uint pid; Native.GetWindowThreadProcessId(h, out pid);
|
|
6018
|
+
if (pid == self) return true;
|
|
5004
6019
|
list.Add(WinDict(h));
|
|
5005
6020
|
return true;
|
|
5006
6021
|
}, IntPtr.Zero);
|
|
5007
|
-
return Dict("windows", list)
|
|
6022
|
+
return Dict("windows", list, "virtualScreen", VirtualScreenDict(),
|
|
6023
|
+
"cooperation", Panic.CooperationState(), "indicator", IndicatorSnapshot());
|
|
6024
|
+
}
|
|
6025
|
+
|
|
6026
|
+
static string IndicatorStateName(int want)
|
|
6027
|
+
{
|
|
6028
|
+
switch (want) {
|
|
6029
|
+
case 0: return "idle";
|
|
6030
|
+
case 1: return "acting";
|
|
6031
|
+
case 2: return "capture";
|
|
6032
|
+
case 3: return "thinking";
|
|
6033
|
+
case 4: return "paused";
|
|
6034
|
+
case 5: return "asking";
|
|
6035
|
+
case 6: return "human-handoff";
|
|
6036
|
+
case 7: return "preparing";
|
|
6037
|
+
case 8: return "reserved";
|
|
6038
|
+
default: return "unknown";
|
|
6039
|
+
}
|
|
6040
|
+
}
|
|
6041
|
+
|
|
6042
|
+
static Dictionary<string, object> IndicatorSnapshot()
|
|
6043
|
+
{
|
|
6044
|
+
// Want/title describe the last rendered frame, not a prediction that
|
|
6045
|
+
// an input will run. The cooperation record supplies the handoff cause.
|
|
6046
|
+
int want = Glow.Want;
|
|
6047
|
+
return Dict("state", IndicatorStateName(want), "want", want,
|
|
6048
|
+
"title", Glow.ShownTitle ?? "", "injecting", Glow.Injecting,
|
|
6049
|
+
"acting", Glow.Acting, "preparing", Glow.Preparing, "holding", Glow.Holding);
|
|
5008
6050
|
}
|
|
5009
6051
|
|
|
5010
6052
|
static IntPtr FindHwnd(Dictionary<string, object> a)
|
|
@@ -5061,15 +6103,17 @@ namespace DshComputerUse.Worker
|
|
|
5061
6103
|
// The whole ladder now lives in RefocusWindow (shared with the keyboard focus
|
|
5062
6104
|
// guard, which must re-assert the same window before typing).
|
|
5063
6105
|
string how = RefocusWindow(h, GetB(a, "clickFallback", true));
|
|
5064
|
-
|
|
6106
|
+
uint confirmedPid = 0;
|
|
6107
|
+
if (how == null || !Native.IsWindow(h) || !IsForeground(h) || (confirmedPid = PidOf(h)) == 0) how = "failed";
|
|
5065
6108
|
else
|
|
5066
6109
|
{
|
|
5067
6110
|
SetSticky(h);
|
|
5068
|
-
_kickoffFocus.Confirm(h.ToInt64(),
|
|
6111
|
+
_kickoffFocus.Confirm(h.ToInt64(), confirmedPid);
|
|
5069
6112
|
}
|
|
5070
6113
|
CheckHumanInput();
|
|
6114
|
+
if (how != "failed") _keyboardReobserveRequired = false;
|
|
5071
6115
|
return Dict("activated", how != "failed", "activatedVia", how,
|
|
5072
|
-
"hwnd", h.ToInt64(),
|
|
6116
|
+
"hwnd", h.ToInt64(), "pid", confirmedPid == 0 ? null : (object)confirmedPid,
|
|
5073
6117
|
"hint", how == "failed"
|
|
5074
6118
|
? "foreground lock refused every method; click the window (or its taskbar button) and retry"
|
|
5075
6119
|
: null);
|
|
@@ -5089,6 +6133,22 @@ namespace DshComputerUse.Worker
|
|
|
5089
6133
|
// 2. if that window belongs to an ELEVATED process while we are not, UIPI silently
|
|
5090
6134
|
// drops every event — say so loudly instead of reporting a successful no-op.
|
|
5091
6135
|
static IntPtr _stickyHwnd = IntPtr.Zero;
|
|
6136
|
+
static volatile bool _keyboardReobserveRequired;
|
|
6137
|
+
const string KeyboardReobserveHint = "A window-management chord was dispatched; this is not proof that a switch succeeded. " +
|
|
6138
|
+
"Inspect computer_state {windows:true}, then use computer_window {hwnd:...,op:'activate'} and require activated:true before further keyboard input.";
|
|
6139
|
+
static void InvalidateKeyboardTarget()
|
|
6140
|
+
{
|
|
6141
|
+
_stickyHwnd = IntPtr.Zero;
|
|
6142
|
+
ResetKickoffFocus();
|
|
6143
|
+
_keyboardReobserveRequired = true;
|
|
6144
|
+
}
|
|
6145
|
+
static void CheckKeyboardReobserve()
|
|
6146
|
+
{
|
|
6147
|
+
if (!_keyboardReobserveRequired) return;
|
|
6148
|
+
var error = NotDispatched("INPUT_REOBSERVE_REQUIRED", "INPUT_REOBSERVE_REQUIRED: " + KeyboardReobserveHint);
|
|
6149
|
+
error.Data["reobserveRequired"] = true;
|
|
6150
|
+
throw error;
|
|
6151
|
+
}
|
|
5092
6152
|
static readonly KickoffFocus _kickoffFocus = new KickoffFocus();
|
|
5093
6153
|
public static void ResetKickoffFocus() { _kickoffFocus.Reset(); }
|
|
5094
6154
|
public static long RecoveryTarget(out uint pid)
|
|
@@ -5108,13 +6168,21 @@ namespace DshComputerUse.Worker
|
|
|
5108
6168
|
IntPtr fg = TopLevel(Native.GetForegroundWindow());
|
|
5109
6169
|
string problem = _kickoffFocus.CaptureProblem(Panic.CycleLit, Panic.Stopped, Panic.Exited,
|
|
5110
6170
|
target != IntPtr.Zero && Native.IsWindow(target), fg.ToInt64(), PidOf(fg), IsHostWindow(fg));
|
|
5111
|
-
if (problem != null)
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
"
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
6171
|
+
if (problem != null)
|
|
6172
|
+
{
|
|
6173
|
+
// NAME THE ACTUAL FOREGROUND (test report 2026-11-04): "identify the intended
|
|
6174
|
+
// window" without saying who holds the foreground forced an extra computer_state
|
|
6175
|
+
// round trip just to learn the one fact this refusal already measured.
|
|
6176
|
+
StringBuilder fgTitle = new StringBuilder(256);
|
|
6177
|
+
if (fg != IntPtr.Zero) Native.GetWindowText(fg, fgTitle, fgTitle.Capacity);
|
|
6178
|
+
throw new Exception(
|
|
6179
|
+
"KICKOFF-FOCUS-REQUIRED: " + problem + ". No screenshot was returned. " +
|
|
6180
|
+
"Foreground now: hwnd=" + fg.ToInt64() + ", pid=" + PidOf(fg) +
|
|
6181
|
+
", title=\"" + fgTitle.ToString() + "\". " +
|
|
6182
|
+
"If the reported foreground HWND and PID match the target already verified for this task, reactivate it. " +
|
|
6183
|
+
"Otherwise identify the intended window with computer_state {windows:true}. " +
|
|
6184
|
+
"Use computer_window {hwnd:...,op:'activate'} and require activated:true before retrying the screenshot.");
|
|
6185
|
+
}
|
|
5118
6186
|
}
|
|
5119
6187
|
|
|
5120
6188
|
static IntPtr TopLevel(IntPtr h)
|
|
@@ -5673,6 +6741,45 @@ namespace DshComputerUse.Worker
|
|
|
5673
6741
|
catch { return false; }
|
|
5674
6742
|
}
|
|
5675
6743
|
|
|
6744
|
+
static int ProcessIntegrityLevel(uint pid)
|
|
6745
|
+
{
|
|
6746
|
+
if (pid == 0) return -1;
|
|
6747
|
+
IntPtr proc = IntPtr.Zero, token = IntPtr.Zero, buffer = IntPtr.Zero;
|
|
6748
|
+
try {
|
|
6749
|
+
proc = Native.OpenProcess(Native.PROCESS_QUERY_LIMITED_INFORMATION, false, pid);
|
|
6750
|
+
if (proc == IntPtr.Zero || !Native.OpenProcessToken(proc, Native.TOKEN_QUERY, out token)) return -1;
|
|
6751
|
+
uint needed;
|
|
6752
|
+
Native.GetTokenInformationBuffer(token, 25, IntPtr.Zero, 0, out needed); // TokenIntegrityLevel
|
|
6753
|
+
if (needed < IntPtr.Size || needed > 4096) return -1;
|
|
6754
|
+
buffer = Marshal.AllocHGlobal((int)needed);
|
|
6755
|
+
if (!Native.GetTokenInformationBuffer(token, 25, buffer, needed, out needed)) return -1;
|
|
6756
|
+
IntPtr sid = Marshal.ReadIntPtr(buffer);
|
|
6757
|
+
if (sid == IntPtr.Zero) return -1;
|
|
6758
|
+
IntPtr countPointer = Native.GetSidSubAuthorityCount(sid);
|
|
6759
|
+
if (countPointer == IntPtr.Zero) return -1;
|
|
6760
|
+
byte count = Marshal.ReadByte(countPointer);
|
|
6761
|
+
if (count == 0 || count > 15) return -1;
|
|
6762
|
+
IntPtr rid = Native.GetSidSubAuthority(sid, (uint)(count - 1));
|
|
6763
|
+
return rid == IntPtr.Zero ? -1 : Marshal.ReadInt32(rid);
|
|
6764
|
+
} catch { return -1; }
|
|
6765
|
+
finally {
|
|
6766
|
+
if (buffer != IntPtr.Zero) Marshal.FreeHGlobal(buffer);
|
|
6767
|
+
if (token != IntPtr.Zero) Native.CloseHandle(token);
|
|
6768
|
+
if (proc != IntPtr.Zero) Native.CloseHandle(proc);
|
|
6769
|
+
}
|
|
6770
|
+
}
|
|
6771
|
+
static bool AsyncKeyIntegrityAllows(int self, int foreground)
|
|
6772
|
+
{ return self >= 0 && foreground >= 0 && foreground <= self; }
|
|
6773
|
+
internal static bool CanObserveAsyncKeys(out IntPtr foreground)
|
|
6774
|
+
{
|
|
6775
|
+
foreground = Native.GetForegroundWindow();
|
|
6776
|
+
if (foreground == IntPtr.Zero) return false;
|
|
6777
|
+
uint pid = PidOf(foreground);
|
|
6778
|
+
// Unknown is NOT unelevated. GetAsyncKeyState reports zero for UIPI
|
|
6779
|
+
// failures too, so only known same/lower integrity is usable evidence.
|
|
6780
|
+
return AsyncKeyIntegrityAllows(ProcessIntegrityLevel((uint)Process.GetCurrentProcess().Id), ProcessIntegrityLevel(pid)) &&
|
|
6781
|
+
Native.GetForegroundWindow() == foreground && PidOf(foreground) == pid;
|
|
6782
|
+
}
|
|
5676
6783
|
static bool ProcessElevated(uint pid)
|
|
5677
6784
|
{
|
|
5678
6785
|
if (pid == 0) return false;
|
|
@@ -5812,6 +6919,7 @@ namespace DshComputerUse.Worker
|
|
|
5812
6919
|
/// reach because of integrity levels.
|
|
5813
6920
|
static Dictionary<string, object> FocusReport(Dictionary<string, object> a, bool refocus)
|
|
5814
6921
|
{
|
|
6922
|
+
if (refocus) CheckKeyboardReobserve(); // explicit old expectHwnd is not fresh authorization
|
|
5815
6923
|
IntPtr want = IntPtr.Zero;
|
|
5816
6924
|
if (a != null && a.ContainsKey("expectHwnd")) want = new IntPtr(GetI(a, "expectHwnd", 0));
|
|
5817
6925
|
else if (a != null && a.ContainsKey("hwnd")) want = new IntPtr(GetI(a, "hwnd", 0));
|
|
@@ -5889,6 +6997,43 @@ namespace DshComputerUse.Worker
|
|
|
5889
6997
|
}
|
|
5890
6998
|
|
|
5891
6999
|
// ---------- UIA ----------
|
|
7000
|
+
static void AddUiGeometry(Dictionary<string, object> d, System.Windows.Rect? bounds,
|
|
7001
|
+
bool? offscreen, Rectangle virtualScreen)
|
|
7002
|
+
{
|
|
7003
|
+
bool valid = false;
|
|
7004
|
+
if (bounds.HasValue) {
|
|
7005
|
+
System.Windows.Rect r = bounds.Value;
|
|
7006
|
+
// Empty UIA rectangles contain infinities. Casting these to int
|
|
7007
|
+
// manufactured INT32_MIN coordinates that looked like real layout.
|
|
7008
|
+
valid = !r.IsEmpty && r.Width > 0 && r.Height > 0 &&
|
|
7009
|
+
!Double.IsNaN(r.X) && !Double.IsNaN(r.Y) &&
|
|
7010
|
+
!Double.IsNaN(r.Width) && !Double.IsNaN(r.Height) &&
|
|
7011
|
+
r.X > Int32.MinValue && r.Y > Int32.MinValue &&
|
|
7012
|
+
r.Right < Int32.MaxValue && r.Bottom < Int32.MaxValue &&
|
|
7013
|
+
r.Width < Int32.MaxValue && r.Height < Int32.MaxValue;
|
|
7014
|
+
if (valid) {
|
|
7015
|
+
long left = (long)Math.Floor(r.Left), top = (long)Math.Floor(r.Top);
|
|
7016
|
+
long right = (long)Math.Ceiling(r.Right), bottom = (long)Math.Ceiling(r.Bottom);
|
|
7017
|
+
valid = left > Int32.MinValue && top > Int32.MinValue &&
|
|
7018
|
+
right - left <= Int32.MaxValue && bottom - top <= Int32.MaxValue;
|
|
7019
|
+
if (valid) {
|
|
7020
|
+
d["rect"] = Dict("x", (int)left, "y", (int)top, "width", (int)(right - left), "height", (int)(bottom - top));
|
|
7021
|
+
d["intersectsVirtualScreen"] = r.Right > virtualScreen.Left && r.Left < virtualScreen.Right &&
|
|
7022
|
+
r.Bottom > virtualScreen.Top && r.Top < virtualScreen.Bottom;
|
|
7023
|
+
}
|
|
7024
|
+
}
|
|
7025
|
+
}
|
|
7026
|
+
d["rectValid"] = valid;
|
|
7027
|
+
d["isOffscreen"] = offscreen.HasValue ? (object)offscreen.Value : null;
|
|
7028
|
+
if (!valid) {
|
|
7029
|
+
d["rect"] = null;
|
|
7030
|
+
d["intersectsVirtualScreen"] = null;
|
|
7031
|
+
d["rectInvalidReason"] = bounds.HasValue ? "empty-or-invalid-provider-bounds" : "provider-bounds-unavailable";
|
|
7032
|
+
}
|
|
7033
|
+
// Neither a valid rectangle nor IsOffscreen=false proves that this
|
|
7034
|
+
// element owns a pixel; use the point probe for occlusion/hit testing.
|
|
7035
|
+
}
|
|
7036
|
+
|
|
5892
7037
|
static Dictionary<string, object> ElemDict(AutomationElement e)
|
|
5893
7038
|
{
|
|
5894
7039
|
Dictionary<string, object> d = new Dictionary<string, object>();
|
|
@@ -5898,10 +7043,15 @@ namespace DshComputerUse.Worker
|
|
|
5898
7043
|
string ctype = "";
|
|
5899
7044
|
try { if (e.Current.ControlType != null) ctype = e.Current.ControlType.ProgrammaticName.Replace("ControlType.", ""); }
|
|
5900
7045
|
catch { }
|
|
5901
|
-
System.Windows.Rect r = e.Current.BoundingRectangle;
|
|
5902
7046
|
d["name"] = name;
|
|
5903
7047
|
d["role"] = ctype;
|
|
5904
|
-
|
|
7048
|
+
System.Windows.Rect? bounds = null;
|
|
7049
|
+
bool? offscreen = null;
|
|
7050
|
+
try { bounds = e.Current.BoundingRectangle; } catch { }
|
|
7051
|
+
try { offscreen = e.Current.IsOffscreen; } catch { }
|
|
7052
|
+
AddUiGeometry(d, bounds, offscreen, new Rectangle(
|
|
7053
|
+
Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN),
|
|
7054
|
+
Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
5905
7055
|
try { d["automationId"] = e.Current.AutomationId; } catch { }
|
|
5906
7056
|
try { d["className"] = e.Current.ClassName; } catch { }
|
|
5907
7057
|
try { d["runtimeId"] = String.Join(".", e.GetRuntimeId()); d["pid"] = e.Current.ProcessId; } catch { }
|
|
@@ -5927,61 +7077,205 @@ namespace DshComputerUse.Worker
|
|
|
5927
7077
|
return d;
|
|
5928
7078
|
}
|
|
5929
7079
|
|
|
7080
|
+
// >>>UIA-QUERY-BUDGET<<<
|
|
7081
|
+
// Cooperative limits between provider calls. A synchronous UIA/COM call
|
|
7082
|
+
// already in progress is NOT interruptible by this traversal budget.
|
|
7083
|
+
internal sealed class UiBudgetStop : Exception { }
|
|
7084
|
+
internal sealed class UiReadBudget
|
|
7085
|
+
{
|
|
7086
|
+
readonly Func<long> clock;
|
|
7087
|
+
readonly long started;
|
|
7088
|
+
readonly List<string> reasons = new List<string>();
|
|
7089
|
+
public readonly int BudgetMs;
|
|
7090
|
+
public int Calls, Scanned;
|
|
7091
|
+
public long MaxCallMs;
|
|
7092
|
+
public UiReadBudget(int budgetMs, Func<long> now)
|
|
7093
|
+
{ BudgetMs = Math.Max(100, Math.Min(5000, budgetMs)); clock = now; started = now(); }
|
|
7094
|
+
public long ElapsedMs { get { return Math.Max(0, clock() - started); } }
|
|
7095
|
+
public bool Truncated { get { return reasons.Count > 0; } }
|
|
7096
|
+
public void Mark(string reason) { if (!reasons.Contains(reason)) reasons.Add(reason); }
|
|
7097
|
+
public bool Continue()
|
|
7098
|
+
{ if (ElapsedMs < BudgetMs) return true; Mark("deadline"); return false; }
|
|
7099
|
+
public T Read<T>(Func<T> read)
|
|
7100
|
+
{
|
|
7101
|
+
if (!Continue()) throw new UiBudgetStop();
|
|
7102
|
+
long before = clock(); Calls++;
|
|
7103
|
+
try {
|
|
7104
|
+
T value = read();
|
|
7105
|
+
if (!Continue()) throw new UiBudgetStop();
|
|
7106
|
+
return value;
|
|
7107
|
+
} finally { MaxCallMs = Math.Max(MaxCallMs, Math.Max(0, clock() - before)); }
|
|
7108
|
+
}
|
|
7109
|
+
public Dictionary<string, object> Report(int maxNodes, int depth)
|
|
7110
|
+
{
|
|
7111
|
+
return Dict("budgetMs", BudgetMs, "elapsedMs", ElapsedMs, "maxNodes", maxNodes, "depth", depth,
|
|
7112
|
+
"scanned", Scanned, "providerCalls", Calls, "maxCallMs", MaxCallMs,
|
|
7113
|
+
"truncatedReasons", reasons.ToArray(), "singleCallInterruptible", false,
|
|
7114
|
+
"note", "Soft traversal budget checked between provider calls; one synchronous UIA/COM call can exceed it. A partial result cannot prove absence or uniqueness.");
|
|
7115
|
+
}
|
|
7116
|
+
}
|
|
7117
|
+
static long UiClockMs()
|
|
7118
|
+
{ return (long)(Stopwatch.GetTimestamp() * (1000.0 / Stopwatch.Frequency)); }
|
|
7119
|
+
static UiReadBudget NewUiBudget(Dictionary<string, object> a)
|
|
7120
|
+
{ return new UiReadBudget(GetI(a, "budgetMs", 1200), UiClockMs); }
|
|
7121
|
+
|
|
7122
|
+
internal static void ValidateUiQueryScope(bool scoped, bool filtered, bool anchored, int depth, int maxNodes)
|
|
7123
|
+
{
|
|
7124
|
+
if (filtered && !scoped)
|
|
7125
|
+
throw new Exception("UIA_SCOPE_REQUIRED: filtered UIA queries require hwnd or an unambiguous window title; use computer_state to identify the window, or computer_uia at for a point probe.");
|
|
7126
|
+
if (!anchored && (depth > 6 || maxNodes > 300))
|
|
7127
|
+
throw new Exception("UIA_ANCHOR_REQUIRED: broad UIA traversal requires an observed anchor point in the requested window; use computer_uia at first, then anchor with depth/maxNodes limits. An exact id does not avoid traversal.");
|
|
7128
|
+
}
|
|
7129
|
+
internal static bool UiAnchorOwned(long requestedTop, uint requestedPid, long actualTop, uint actualPid)
|
|
7130
|
+
{ return requestedTop != 0 && requestedPid != 0 && requestedTop == actualTop && requestedPid == actualPid; }
|
|
7131
|
+
static AutomationElement UiQueryRoot(Dictionary<string, object> a, IntPtr root, UiReadBudget budget)
|
|
7132
|
+
{
|
|
7133
|
+
string anchor = GetS(a, "anchor", null);
|
|
7134
|
+
if (String.IsNullOrWhiteSpace(anchor)) return budget.Read(delegate { return AutomationElement.FromHandle(root); });
|
|
7135
|
+
string[] xy = anchor.Split(','); int x, y;
|
|
7136
|
+
if (xy.Length != 2 || !Int32.TryParse(xy[0].Trim(), out x) || !Int32.TryParse(xy[1].Trim(), out y))
|
|
7137
|
+
throw new Exception("UIA_ANCHOR_INVALID: anchor must be physical screen x,y");
|
|
7138
|
+
POINT point; point.X = x; point.Y = y;
|
|
7139
|
+
uint pid = PidOf(root); IntPtr expectedTop = TopLevel(root), hit = Native.WindowFromPoint(point);
|
|
7140
|
+
if (!Native.IsWindow(root) || !UiAnchorOwned(expectedTop.ToInt64(), pid, TopLevel(hit).ToInt64(), PidOf(hit)))
|
|
7141
|
+
throw new Exception("UIA_ANCHOR_MISMATCH: anchor is not currently owned by the requested window and process; no provider query was started");
|
|
7142
|
+
AutomationElement element = budget.Read(delegate { return AutomationElement.FromPoint(new System.Windows.Point(x, y)); });
|
|
7143
|
+
IntPtr afterHit = Native.WindowFromPoint(point);
|
|
7144
|
+
if (element == null || budget.Read(delegate { return element.Current.ProcessId; }) != (int)pid ||
|
|
7145
|
+
PidOf(root) != pid || !UiAnchorOwned(expectedTop.ToInt64(), pid, TopLevel(afterHit).ToInt64(), PidOf(afterHit)))
|
|
7146
|
+
throw new Exception("UIA_ANCHOR_MISMATCH: provider hit or window changed during the point probe; repeat the observation");
|
|
7147
|
+
return element;
|
|
7148
|
+
}
|
|
7149
|
+
|
|
7150
|
+
// Generic only so deterministic fake trees exercise the identical walk.
|
|
7151
|
+
// Child/sibling navigation is incremental: no FindAll collection can be
|
|
7152
|
+
// built before the client-side node/depth/deadline checks run.
|
|
7153
|
+
internal static List<T> UiWalkBounded<T>(T start, int depth, int maxNodes, int limit, UiReadBudget budget,
|
|
7154
|
+
Func<T, bool> matches, Func<T, T> firstChild, Func<T, T> nextSibling) where T : class
|
|
7155
|
+
{
|
|
7156
|
+
var found = new List<T>(); var queue = new Queue<KeyValuePair<T, int>>();
|
|
7157
|
+
if (start == null) return found;
|
|
7158
|
+
// A queue entry is a parent whose direct children form the next BFS
|
|
7159
|
+
// segment. Visit each child before asking for its next sibling: a
|
|
7160
|
+
// wide provider can no longer consume the budget before any match
|
|
7161
|
+
// in that sibling list has been inspected/materialized.
|
|
7162
|
+
queue.Enqueue(new KeyValuePair<T, int>(start, -1));
|
|
7163
|
+
while (queue.Count > 0 && budget.Scanned < maxNodes && budget.Continue())
|
|
7164
|
+
{
|
|
7165
|
+
var segment = queue.Dequeue(); int level = segment.Value + 1;
|
|
7166
|
+
try
|
|
7167
|
+
{
|
|
7168
|
+
T node = segment.Value < 0 ? segment.Key : budget.Read(delegate { return firstChild(segment.Key); });
|
|
7169
|
+
while (node != null && budget.Continue())
|
|
7170
|
+
{
|
|
7171
|
+
if (budget.Scanned >= maxNodes) { budget.Mark("max-nodes"); return found; }
|
|
7172
|
+
budget.Scanned++;
|
|
7173
|
+
try {
|
|
7174
|
+
if (matches(node)) {
|
|
7175
|
+
found.Add(node);
|
|
7176
|
+
if (found.Count >= limit) { budget.Mark("match-limit"); return found; }
|
|
7177
|
+
}
|
|
7178
|
+
if (!budget.Continue()) return found;
|
|
7179
|
+
if (level < depth) queue.Enqueue(new KeyValuePair<T, int>(node, level));
|
|
7180
|
+
else if (budget.Read(delegate { return firstChild(node); }) != null) budget.Mark("depth");
|
|
7181
|
+
} catch (UiBudgetStop) { return found; }
|
|
7182
|
+
catch { budget.Mark("provider-error"); }
|
|
7183
|
+
if (segment.Value < 0) break;
|
|
7184
|
+
if (budget.Scanned >= maxNodes) { budget.Mark("max-nodes"); return found; }
|
|
7185
|
+
T previous = node;
|
|
7186
|
+
node = budget.Read(delegate { return nextSibling(previous); });
|
|
7187
|
+
}
|
|
7188
|
+
}
|
|
7189
|
+
catch (UiBudgetStop) { break; }
|
|
7190
|
+
catch { budget.Mark("provider-error"); }
|
|
7191
|
+
}
|
|
7192
|
+
if (queue.Count > 0) budget.Mark(budget.Scanned >= maxNodes ? "max-nodes" : "pending-nodes");
|
|
7193
|
+
return found;
|
|
7194
|
+
}
|
|
7195
|
+
// <<<UIA-QUERY-BUDGET<<<
|
|
7196
|
+
|
|
7197
|
+
static Dictionary<string, object> ElemDictBudgeted(AutomationElement e, UiReadBudget budget)
|
|
7198
|
+
{
|
|
7199
|
+
var d = new Dictionary<string, object>();
|
|
7200
|
+
string name = "", role = ""; System.Windows.Rect? bounds = null; bool? offscreen = null;
|
|
7201
|
+
try { name = budget.Read(delegate { return e.Current.Name; }) ?? ""; } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7202
|
+
try { ControlType ct = budget.Read(delegate { return e.Current.ControlType; }); if (ct != null) role = ct.ProgrammaticName.Replace("ControlType.", ""); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7203
|
+
d["name"] = name.Length > 160 ? name.Substring(0, 160) : name; d["role"] = role;
|
|
7204
|
+
try { bounds = budget.Read(delegate { return e.Current.BoundingRectangle; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7205
|
+
try { offscreen = budget.Read(delegate { return e.Current.IsOffscreen; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7206
|
+
AddUiGeometry(d, bounds, offscreen, new Rectangle(Native.GetSystemMetrics(Native.SM_XVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_YVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CXVIRTUALSCREEN), Native.GetSystemMetrics(Native.SM_CYVIRTUALSCREEN)));
|
|
7207
|
+
try { d["automationId"] = budget.Read(delegate { return e.Current.AutomationId; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7208
|
+
try { d["className"] = budget.Read(delegate { return e.Current.ClassName; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7209
|
+
try { d["runtimeId"] = String.Join(".", budget.Read(delegate { return e.GetRuntimeId(); })); d["pid"] = budget.Read(delegate { return e.Current.ProcessId; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7210
|
+
try { d["isPassword"] = budget.Read(delegate { return e.Current.IsPassword; }); } catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7211
|
+
try {
|
|
7212
|
+
var patterns = new List<string>();
|
|
7213
|
+
foreach (AutomationPattern pattern in budget.Read(delegate { return e.GetSupportedPatterns(); })) {
|
|
7214
|
+
string text = pattern.ProgrammaticName.Replace("PatternIdentifiers", "").Replace("Pattern", "").TrimEnd('.', ' ');
|
|
7215
|
+
if (text.Length > 0) patterns.Add(text);
|
|
7216
|
+
}
|
|
7217
|
+
d["patterns"] = patterns;
|
|
7218
|
+
} catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7219
|
+
try {
|
|
7220
|
+
object value = budget.Read(delegate { object p; return e.TryGetCurrentPattern(ValuePattern.Pattern, out p) ? p : null; });
|
|
7221
|
+
if (value != null) d["value"] = budget.Read(delegate { return ((ValuePattern)value).Current.Value; });
|
|
7222
|
+
} catch (UiBudgetStop) { throw; } catch { budget.Mark("provider-error"); }
|
|
7223
|
+
return d;
|
|
7224
|
+
}
|
|
7225
|
+
|
|
5930
7226
|
static Dictionary<string, object> Uia(Dictionary<string, object> a)
|
|
5931
7227
|
{
|
|
5932
7228
|
IntPtr root = FindHwnd(a);
|
|
5933
|
-
AutomationElement rootEl = root != IntPtr.Zero ? AutomationElement.FromHandle(root) : AutomationElement.RootElement;
|
|
5934
7229
|
int maxDepth = Math.Max(1, Math.Min(24, GetI(a, "depth", 6)));
|
|
5935
|
-
int maxNodes = Math.Max(10, Math.Min(1200, GetI(a, "maxNodes",
|
|
5936
|
-
|
|
7230
|
+
int maxNodes = Math.Max(10, Math.Min(1200, GetI(a, "maxNodes", 120)));
|
|
7231
|
+
bool query = GetB(a, "query", false), anchored = !String.IsNullOrWhiteSpace(GetS(a, "anchor", null));
|
|
7232
|
+
ValidateUiQueryScope(root != IntPtr.Zero, query, anchored, maxDepth, maxNodes);
|
|
7233
|
+
if (root == IntPtr.Zero) root = Native.GetForegroundWindow();
|
|
7234
|
+
if (root == IntPtr.Zero) throw new Exception("UIA_SCOPE_REQUIRED: no foreground window; specify hwnd");
|
|
7235
|
+
UiReadBudget budget = NewUiBudget(a);
|
|
7236
|
+
AutomationElement rootEl;
|
|
7237
|
+
try { rootEl = UiQueryRoot(a, root, budget); }
|
|
7238
|
+
catch (UiBudgetStop) { return Dict("flat", new List<object>(), "count", 0, "scanned", 0, "truncated", true, "budget", budget.Report(maxNodes, maxDepth), "hwnd", root.ToInt64(), "pid", PidOf(root)); }
|
|
7239
|
+
if (query)
|
|
5937
7240
|
{
|
|
5938
|
-
int scanned; bool truncated;
|
|
5939
|
-
List<AutomationElement> matches = FindUiMatches(rootEl, GetS(a, "nameContains", null),
|
|
5940
|
-
GetS(a, "automationId", null), GetS(a, "role", null), maxDepth, maxNodes,
|
|
5941
|
-
Math.Max(1, Math.Min(80, GetI(a, "limit", 80))), out scanned, out truncated);
|
|
5942
7241
|
List<object> selected = new List<object>();
|
|
5943
7242
|
string witness = root != IntPtr.Zero && GetB(a, "targets", false) ? UiWindowWitness(root) : null;
|
|
5944
7243
|
var ancestorCache = new Dictionary<string, List<object>>();
|
|
5945
|
-
|
|
5946
|
-
{
|
|
5947
|
-
try {
|
|
5948
|
-
|
|
7244
|
+
AutomationElement evidenceRoot = rootEl;
|
|
7245
|
+
if (witness != null && anchored) {
|
|
7246
|
+
try { evidenceRoot = budget.Read(delegate { return AutomationElement.FromHandle(root); }); }
|
|
7247
|
+
catch (UiBudgetStop) { evidenceRoot = null; }
|
|
7248
|
+
}
|
|
7249
|
+
string nameFilter = GetS(a, "nameContains", null), idFilter = GetS(a, "automationId", null), roleFilter = GetS(a, "role", null);
|
|
7250
|
+
CacheRequest selectorCache = UiSelectorCache();
|
|
7251
|
+
UiWalkBounded(rootEl, maxDepth, maxNodes, Math.Max(1, Math.Min(80, GetI(a, "limit", 80))), budget,
|
|
7252
|
+
delegate(AutomationElement match) {
|
|
7253
|
+
if (!ElementMatchesBudgeted(match, nameFilter, idFilter, roleFilter, budget)) return false;
|
|
7254
|
+
var item = ElemDictBudgeted(match, budget);
|
|
5949
7255
|
if (witness != null) {
|
|
5950
|
-
try {
|
|
7256
|
+
try {
|
|
7257
|
+
if (evidenceRoot == null) throw new UiBudgetStop();
|
|
7258
|
+
item["target"] = RememberUiTarget(match, root, witness, evidenceRoot, ancestorCache, item, budget);
|
|
7259
|
+
}
|
|
7260
|
+
catch (UiBudgetStop) { item["targetUnavailable"] = true; }
|
|
5951
7261
|
catch { item["targetUnavailable"] = true; }
|
|
5952
7262
|
}
|
|
5953
7263
|
selected.Add(item);
|
|
5954
|
-
|
|
5955
|
-
|
|
7264
|
+
return true;
|
|
7265
|
+
},
|
|
7266
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element, selectorCache); },
|
|
7267
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element, selectorCache); });
|
|
5956
7268
|
if (witness != null && witness != UiWindowWitness(root)) throw new Exception("TARGET_STALE: window changed during observation; retry the query");
|
|
5957
|
-
return Dict("flat", selected, "count", selected.Count, "scanned",
|
|
5958
|
-
"hwnd", root.ToInt64(), "pid", root
|
|
7269
|
+
return Dict("flat", selected, "count", selected.Count, "scanned", budget.Scanned, "truncated", budget.Truncated,
|
|
7270
|
+
"hwnd", root.ToInt64(), "pid", PidOf(root), "budget", budget.Report(maxNodes, maxDepth));
|
|
5959
7271
|
}
|
|
5960
7272
|
List<object> flat = new List<object>();
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
KeyValuePair<AutomationElement, int> kv = q.Dequeue();
|
|
5968
|
-
try { flat.Add(ElemDict(kv.Key)); }
|
|
5969
|
-
catch { treeTruncated = true; continue; }
|
|
5970
|
-
if (kv.Value < maxDepth)
|
|
5971
|
-
{
|
|
5972
|
-
try
|
|
5973
|
-
{
|
|
5974
|
-
foreach (AutomationElement c in kv.Key.FindAll(TreeScope.Children, Condition.TrueCondition))
|
|
5975
|
-
{
|
|
5976
|
-
if (flat.Count + q.Count >= maxNodes) { treeTruncated = true; break; }
|
|
5977
|
-
q.Enqueue(new KeyValuePair<AutomationElement, int>(c, kv.Value + 1));
|
|
5978
|
-
}
|
|
5979
|
-
}
|
|
5980
|
-
catch { treeTruncated = true; }
|
|
5981
|
-
}
|
|
5982
|
-
else try { if (kv.Key.FindFirst(TreeScope.Children, Condition.TrueCondition) != null) treeTruncated = true; } catch { treeTruncated = true; }
|
|
5983
|
-
}
|
|
5984
|
-
return Dict("root", ElemDict(rootEl), "flat", flat, "count", flat.Count, "truncated", treeTruncated || q.Count > 0);
|
|
7273
|
+
UiWalkBounded(rootEl, maxDepth, maxNodes, maxNodes + 1, budget,
|
|
7274
|
+
delegate(AutomationElement element) { flat.Add(ElemDictBudgeted(element, budget)); return false; },
|
|
7275
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element); },
|
|
7276
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element); });
|
|
7277
|
+
return Dict("root", flat.Count > 0 ? flat[0] : null, "flat", flat, "count", flat.Count,
|
|
7278
|
+
"truncated", budget.Truncated, "budget", budget.Report(maxNodes, maxDepth), "hwnd", root.ToInt64(), "pid", PidOf(root));
|
|
5985
7279
|
}
|
|
5986
7280
|
|
|
5987
7281
|
// Observed native identities are private to this worker, bounded, and never reconstructed
|
|
@@ -6005,31 +7299,35 @@ namespace DshComputerUse.Worker
|
|
|
6005
7299
|
return Json.Serialize(new object[] { hwnd.ToInt64(), pid, started, WinTitleOf(hwnd), ClassOf(hwnd), rect.Left, rect.Top, rect.Right, rect.Bottom });
|
|
6006
7300
|
}
|
|
6007
7301
|
static string UiText(string text) { return (text ?? "").Normalize(NormalizationForm.FormKC).Trim(); }
|
|
6008
|
-
static
|
|
7302
|
+
static T UiReadMaybe<T>(UiReadBudget budget, Func<T> read)
|
|
7303
|
+
{ return budget == null ? read() : budget.Read(read); }
|
|
7304
|
+
static object UiSemanticPart(AutomationElement element, UiReadBudget budget = null)
|
|
6009
7305
|
{
|
|
6010
|
-
return new object[] {
|
|
7306
|
+
return new object[] { UiReadMaybe(budget, delegate { return element.Current.ControlType; }).ProgrammaticName,
|
|
7307
|
+
UiText(UiReadMaybe(budget, delegate { return element.Current.Name; })),
|
|
7308
|
+
UiReadMaybe(budget, delegate { return element.Current.AutomationId; }), UiReadMaybe(budget, delegate { return element.Current.ClassName; }) };
|
|
6011
7309
|
}
|
|
6012
|
-
static string UiTargetEvidence(AutomationElement element, IntPtr hwnd)
|
|
7310
|
+
static string UiTargetEvidence(AutomationElement element, IntPtr hwnd, UiReadBudget budget = null)
|
|
6013
7311
|
{
|
|
6014
|
-
return UiTargetEvidence(element, AutomationElement.FromHandle(hwnd), new Dictionary<string, List<object>>(), null);
|
|
7312
|
+
return UiTargetEvidence(element, UiReadMaybe(budget, delegate { return AutomationElement.FromHandle(hwnd); }), new Dictionary<string, List<object>>(), null, budget);
|
|
6015
7313
|
}
|
|
6016
|
-
static string UiTargetEvidence(AutomationElement element, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item)
|
|
7314
|
+
static string UiTargetEvidence(AutomationElement element, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item, UiReadBudget budget = null)
|
|
6017
7315
|
{
|
|
6018
7316
|
var ancestors = new List<object>();
|
|
6019
|
-
AutomationElement current = TreeWalker.ControlViewWalker.GetParent(element);
|
|
6020
|
-
string parentId = current == null ? "" : String.Join(".", current.GetRuntimeId());
|
|
7317
|
+
AutomationElement current = UiReadMaybe(budget, delegate { return TreeWalker.ControlViewWalker.GetParent(element); });
|
|
7318
|
+
string parentId = current == null ? "" : String.Join(".", UiReadMaybe(budget, delegate { return current.GetRuntimeId(); }));
|
|
6021
7319
|
if (parentId.Length == 0 || !cache.TryGetValue(parentId, out ancestors)) {
|
|
6022
7320
|
ancestors = new List<object>();
|
|
6023
|
-
bool rooted = Automation.Compare(element, root);
|
|
7321
|
+
bool rooted = UiReadMaybe(budget, delegate { return Automation.Compare(element, root); });
|
|
6024
7322
|
for (int depth = 0; current != null && depth < 24; depth++) {
|
|
6025
|
-
ancestors.Add(UiSemanticPart(current));
|
|
6026
|
-
if (Automation.Compare(current, root)) { rooted = true; break; }
|
|
6027
|
-
current = TreeWalker.ControlViewWalker.GetParent(current);
|
|
7323
|
+
ancestors.Add(UiSemanticPart(current, budget));
|
|
7324
|
+
if (UiReadMaybe(budget, delegate { return Automation.Compare(current, root); })) { rooted = true; break; }
|
|
7325
|
+
current = UiReadMaybe(budget, delegate { return TreeWalker.ControlViewWalker.GetParent(current); });
|
|
6028
7326
|
}
|
|
6029
7327
|
if (!rooted) throw new Exception("TARGET_STALE: control is outside the observed window or ancestry is incomplete");
|
|
6030
7328
|
if (parentId.Length > 0) cache[parentId] = ancestors;
|
|
6031
7329
|
}
|
|
6032
|
-
if (item == null) item = ElemDict(element);
|
|
7330
|
+
if (item == null) item = budget == null ? ElemDict(element) : ElemDictBudgeted(element, budget);
|
|
6033
7331
|
var patterns = new List<string>();
|
|
6034
7332
|
object raw;
|
|
6035
7333
|
if (!item.TryGetValue("patterns", out raw)) throw new Exception("TARGET_STALE: native patterns unavailable");
|
|
@@ -6037,14 +7335,14 @@ namespace DshComputerUse.Worker
|
|
|
6037
7335
|
patterns.Sort(StringComparer.Ordinal);
|
|
6038
7336
|
// AutomationId is a separate identity channel, so a unique semantic replacement can
|
|
6039
7337
|
// still be recognized if a provider regenerates IDs. Ancestor identities remain strict.
|
|
6040
|
-
return Json.Serialize(new object[] { item["role"], UiText(element.Current.Name),
|
|
7338
|
+
return Json.Serialize(new object[] { item["role"], UiText(UiReadMaybe(budget, delegate { return element.Current.Name; })),
|
|
6041
7339
|
item["className"], item["isPassword"], patterns, ancestors });
|
|
6042
7340
|
}
|
|
6043
|
-
static string RememberUiTarget(AutomationElement element, IntPtr hwnd, string witness, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item)
|
|
7341
|
+
static string RememberUiTarget(AutomationElement element, IntPtr hwnd, string witness, AutomationElement root, Dictionary<string, List<object>> cache, Dictionary<string, object> item, UiReadBudget budget = null)
|
|
6044
7342
|
{
|
|
6045
7343
|
string runtimeId = GetS(item, "runtimeId", "");
|
|
6046
7344
|
if (runtimeId.Length == 0) throw new Exception("native control has no runtime identity");
|
|
6047
|
-
string evidence = UiTargetEvidence(element, root, cache, item);
|
|
7345
|
+
string evidence = UiTargetEvidence(element, root, cache, item, budget);
|
|
6048
7346
|
string autoId = GetS(item, "automationId", "");
|
|
6049
7347
|
string key = Json.Serialize(new object[] { witness, runtimeId, evidence, autoId });
|
|
6050
7348
|
string existing; ObservedUiTarget old;
|
|
@@ -6053,7 +7351,7 @@ namespace DshComputerUse.Worker
|
|
|
6053
7351
|
}
|
|
6054
7352
|
string token = "T" + Guid.NewGuid().ToString("N");
|
|
6055
7353
|
var target = new ObservedUiTarget { Token = token, Key = key, Hwnd = hwnd, Witness = witness, RuntimeId = runtimeId,
|
|
6056
|
-
Evidence = evidence, AutoId = autoId, Name = element.Current.Name,
|
|
7354
|
+
Evidence = evidence, AutoId = autoId, Name = UiReadMaybe(budget, delegate { return element.Current.Name; }),
|
|
6057
7355
|
Role = GetS(item, "role", ""), Element = element, At = NowMs() };
|
|
6058
7356
|
UiTargets[token] = target; UiTargetKeys[key] = token; UiTargetOrder.Enqueue(token);
|
|
6059
7357
|
while (UiTargets.Count > 512) {
|
|
@@ -6071,36 +7369,38 @@ namespace DshComputerUse.Worker
|
|
|
6071
7369
|
throw NotDispatched("TARGET_STALE", "TARGET_STALE: window identity, title or bounds changed; query again; no action sent");
|
|
6072
7370
|
return target;
|
|
6073
7371
|
}
|
|
6074
|
-
static AutomationElement ResolveUiTarget(ObservedUiTarget target, bool rebind, int depth, int budget, out string mode, out int seen
|
|
7372
|
+
static AutomationElement ResolveUiTarget(ObservedUiTarget target, bool rebind, int depth, int budget, out string mode, out int seen,
|
|
7373
|
+
UiReadBudget readBudget = null, Dictionary<string, object> scope = null)
|
|
6075
7374
|
{
|
|
7375
|
+
if (readBudget == null) readBudget = new UiReadBudget(1200, UiClockMs);
|
|
6076
7376
|
mode = "runtime-id"; seen = 0;
|
|
6077
7377
|
try {
|
|
6078
|
-
if (String.Join(".", target.Element.GetRuntimeId()) == target.RuntimeId &&
|
|
6079
|
-
target.Element.Current.AutomationId == target.AutoId && UiTargetEvidence(target.Element, target.Hwnd) == target.Evidence)
|
|
7378
|
+
if (String.Join(".", readBudget.Read(delegate { return target.Element.GetRuntimeId(); })) == target.RuntimeId &&
|
|
7379
|
+
readBudget.Read(delegate { return target.Element.Current.AutomationId; }) == target.AutoId && UiTargetEvidence(target.Element, target.Hwnd, readBudget) == target.Evidence)
|
|
6080
7380
|
return target.Element;
|
|
6081
|
-
} catch { }
|
|
7381
|
+
} catch (UiBudgetStop) { throw; } catch { }
|
|
6082
7382
|
if (!rebind) throw new Exception("TARGET_STALE: observed control was replaced or changed; query again or explicitly request rebind; no action sent");
|
|
6083
|
-
var start = AutomationElement.FromHandle(target.Hwnd);
|
|
7383
|
+
var start = scope == null ? readBudget.Read(delegate { return AutomationElement.FromHandle(target.Hwnd); }) : UiQueryRoot(scope, target.Hwnd, readBudget);
|
|
6084
7384
|
bool truncated; int visited;
|
|
6085
7385
|
List<AutomationElement> candidates = new List<AutomationElement>();
|
|
6086
7386
|
if (!String.IsNullOrEmpty(target.AutoId)) {
|
|
6087
|
-
candidates =
|
|
7387
|
+
candidates = FindUiMatchesBudgeted(start, null, target.AutoId, null, depth, budget, budget, readBudget, out visited, out truncated);
|
|
6088
7388
|
seen += visited;
|
|
6089
7389
|
if (truncated) throw new Exception("INCOMPLETE_UIA_SEARCH: cannot prove unique target recovery; no action sent");
|
|
6090
7390
|
if (candidates.Count > 1) throw new Exception("AMBIGUOUS_UIA_TARGET: native identity matches multiple controls; no action sent");
|
|
6091
7391
|
if (candidates.Count == 1) {
|
|
6092
|
-
if (UiTargetEvidence(candidates[0], target.Hwnd) != target.Evidence)
|
|
7392
|
+
if (UiTargetEvidence(candidates[0], target.Hwnd, readBudget) != target.Evidence)
|
|
6093
7393
|
throw new Exception("TARGET_SEMANTICS_CHANGED: native ID now names a different control/context; no action sent");
|
|
6094
7394
|
mode = "native-identifier";
|
|
6095
7395
|
}
|
|
6096
7396
|
}
|
|
6097
7397
|
if (candidates.Count == 0) {
|
|
6098
7398
|
if (String.IsNullOrWhiteSpace(target.Name)) throw new Exception("TARGET_STALE: target has no usable semantic name; no action sent");
|
|
6099
|
-
var matches =
|
|
7399
|
+
var matches = FindUiMatchesBudgeted(start, target.Name, null, target.Role, depth, budget, budget, readBudget, out visited, out truncated);
|
|
6100
7400
|
seen += visited;
|
|
6101
7401
|
if (truncated) throw new Exception("INCOMPLETE_UIA_SEARCH: cannot prove unique semantic recovery; no action sent");
|
|
6102
7402
|
foreach (AutomationElement element in matches)
|
|
6103
|
-
if (UiTargetEvidence(element, target.Hwnd) == target.Evidence) candidates.Add(element);
|
|
7403
|
+
if (UiTargetEvidence(element, target.Hwnd, readBudget) == target.Evidence) candidates.Add(element);
|
|
6104
7404
|
if (candidates.Count > 1) throw new Exception("AMBIGUOUS_UIA_TARGET: semantic recovery matches multiple controls; no action sent");
|
|
6105
7405
|
if (candidates.Count == 0) throw new Exception("TARGET_STALE: no control retains the observed identity/context; no action sent");
|
|
6106
7406
|
mode = "semantic";
|
|
@@ -6118,9 +7418,28 @@ namespace DshComputerUse.Worker
|
|
|
6118
7418
|
// button, expanding a dropdown, filling a field.
|
|
6119
7419
|
static bool ElementMatches(AutomationElement e, string nameFilter, string autoIdFilter, string roleFilter)
|
|
6120
7420
|
{
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
|
|
7421
|
+
return ElementMatchesBudgeted(e, nameFilter, autoIdFilter, roleFilter, null);
|
|
7422
|
+
}
|
|
7423
|
+
static CacheRequest UiSelectorCache()
|
|
7424
|
+
{
|
|
7425
|
+
// Only the current navigated element, never a cached subtree. This
|
|
7426
|
+
// batches the three selector properties into the navigation call.
|
|
7427
|
+
var cache = new CacheRequest(); cache.TreeScope = TreeScope.Element;
|
|
7428
|
+
cache.AutomationElementMode = AutomationElementMode.Full;
|
|
7429
|
+
cache.Add(AutomationElement.NameProperty); cache.Add(AutomationElement.AutomationIdProperty);
|
|
7430
|
+
cache.Add(AutomationElement.ControlTypeProperty); return cache;
|
|
7431
|
+
}
|
|
7432
|
+
static T UiSelectorRead<T>(UiReadBudget budget, Func<T> cached, Func<T> current)
|
|
7433
|
+
{
|
|
7434
|
+
if (budget != null && !budget.Continue()) throw new UiBudgetStop();
|
|
7435
|
+
try { return cached(); }
|
|
7436
|
+
catch (InvalidOperationException) { return UiReadMaybe(budget, current); }
|
|
7437
|
+
}
|
|
7438
|
+
static bool ElementMatchesBudgeted(AutomationElement e, string nameFilter, string autoIdFilter, string roleFilter, UiReadBudget budget)
|
|
7439
|
+
{
|
|
7440
|
+
if (autoIdFilter != null && !String.Equals(UiSelectorRead(budget, delegate { return e.Cached.AutomationId; }, delegate { return e.Current.AutomationId; }), autoIdFilter, StringComparison.Ordinal)) return false;
|
|
7441
|
+
if (roleFilter != null && !String.Equals(UiSelectorRead(budget, delegate { return e.Cached.ControlType; }, delegate { return e.Current.ControlType; }).ProgrammaticName.Replace("ControlType.", ""), roleFilter, StringComparison.OrdinalIgnoreCase)) return false;
|
|
7442
|
+
if (nameFilter != null && (UiSelectorRead(budget, delegate { return e.Cached.Name; }, delegate { return e.Current.Name; }) ?? "").IndexOf(nameFilter, StringComparison.OrdinalIgnoreCase) < 0) return false;
|
|
6124
7443
|
return true;
|
|
6125
7444
|
}
|
|
6126
7445
|
|
|
@@ -6129,32 +7448,19 @@ namespace DshComputerUse.Worker
|
|
|
6129
7448
|
static List<AutomationElement> FindUiMatches(AutomationElement start, string name, string autoId,
|
|
6130
7449
|
string role, int depth, int maxNodes, int limit, out int seen, out bool truncated)
|
|
6131
7450
|
{
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
}
|
|
6146
|
-
if (kv.Value < depth)
|
|
6147
|
-
foreach (AutomationElement child in kv.Key.FindAll(TreeScope.Children, Condition.TrueCondition))
|
|
6148
|
-
{
|
|
6149
|
-
if (seen + q.Count >= maxNodes) { truncated = true; break; }
|
|
6150
|
-
q.Enqueue(new KeyValuePair<AutomationElement, int>(child, kv.Value + 1));
|
|
6151
|
-
}
|
|
6152
|
-
else if (kv.Key.FindFirst(TreeScope.Children, Condition.TrueCondition) != null)
|
|
6153
|
-
truncated = true;
|
|
6154
|
-
}
|
|
6155
|
-
catch { truncated = true; }
|
|
6156
|
-
}
|
|
6157
|
-
if (q.Count > 0) truncated = true;
|
|
7451
|
+
return FindUiMatchesBudgeted(start, name, autoId, role, depth, maxNodes, limit,
|
|
7452
|
+
new UiReadBudget(1200, UiClockMs), out seen, out truncated);
|
|
7453
|
+
}
|
|
7454
|
+
static List<AutomationElement> FindUiMatchesBudgeted(AutomationElement start, string name, string autoId,
|
|
7455
|
+
string role, int depth, int maxNodes, int limit, UiReadBudget budget, out int seen, out bool truncated)
|
|
7456
|
+
{
|
|
7457
|
+
int before = budget.Scanned;
|
|
7458
|
+
CacheRequest selectorCache = UiSelectorCache();
|
|
7459
|
+
var matches = UiWalkBounded(start, depth, maxNodes, limit, budget,
|
|
7460
|
+
delegate(AutomationElement element) { return ElementMatchesBudgeted(element, name, autoId, role, budget); },
|
|
7461
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetFirstChild(element, selectorCache); },
|
|
7462
|
+
delegate(AutomationElement element) { return TreeWalker.RawViewWalker.GetNextSibling(element, selectorCache); });
|
|
7463
|
+
seen = budget.Scanned - before; truncated = budget.Truncated;
|
|
6158
7464
|
return matches;
|
|
6159
7465
|
}
|
|
6160
7466
|
|
|
@@ -6168,35 +7474,46 @@ namespace DshComputerUse.Worker
|
|
|
6168
7474
|
IntPtr root = observed == null ? FindHwnd(a) : observed.Hwnd;
|
|
6169
7475
|
if (observed != null && a.ContainsKey("hwnd") && FindHwnd(a) != root) throw new Exception("TARGET_STALE: target belongs to another window; no action sent");
|
|
6170
7476
|
if (root != IntPtr.Zero) HostGuard(root, "uiaAct");
|
|
6171
|
-
AutomationElement start = root != IntPtr.Zero ? AutomationElement.FromHandle(root) : AutomationElement.RootElement;
|
|
6172
7477
|
string name = GetS(a, "nameContains", null);
|
|
6173
7478
|
string autoId = GetS(a, "automationId", null);
|
|
6174
7479
|
string role = GetS(a, "role", null);
|
|
6175
7480
|
int index = Math.Max(0, GetI(a, "index", 0));
|
|
6176
|
-
int depth = Math.Max(1, Math.Min(24, GetI(a, "depth",
|
|
6177
|
-
int maxNodes = Math.Max(20, Math.Min(
|
|
7481
|
+
int depth = Math.Max(1, Math.Min(24, GetI(a, "depth", 6)));
|
|
7482
|
+
int maxNodes = Math.Max(20, Math.Min(1200, GetI(a, "maxNodes", 120)));
|
|
7483
|
+
bool rebind = GetB(a, "allowRebind", false);
|
|
7484
|
+
if (observed == null || rebind) ValidateUiQueryScope(root != IntPtr.Zero, true,
|
|
7485
|
+
!String.IsNullOrWhiteSpace(GetS(a, "anchor", null)), depth, maxNodes);
|
|
7486
|
+
UiReadBudget readBudget = NewUiBudget(a);
|
|
6178
7487
|
bool unique = GetB(a, "requireUnique", !a.ContainsKey("index"));
|
|
6179
7488
|
List<object> skipped = new List<object>();
|
|
6180
7489
|
int seen = 0; bool truncated = false; string resolution = null;
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
|
|
7490
|
+
List<AutomationElement> matches;
|
|
7491
|
+
try {
|
|
7492
|
+
matches = observed == null ? FindUiMatchesBudgeted(UiQueryRoot(a, root, readBudget), name, autoId, role, depth, maxNodes,
|
|
7493
|
+
unique ? 2 : index + 1, readBudget, out seen, out truncated) : new List<AutomationElement> {
|
|
7494
|
+
ResolveUiTarget(observed, rebind, depth, maxNodes, out resolution, out seen, readBudget, a) };
|
|
7495
|
+
} catch (UiBudgetStop) {
|
|
7496
|
+
throw new Exception("INCOMPLETE_UIA_SEARCH: soft provider-read budget exhausted; no action sent. Observe a narrower point/anchor or refresh once; a synchronous COM call cannot be interrupted by this budget.");
|
|
7497
|
+
}
|
|
7498
|
+
truncated = truncated || readBudget.Truncated;
|
|
7499
|
+
if (!readBudget.Continue())
|
|
7500
|
+
throw new Exception("INCOMPLETE_UIA_SEARCH: provider-read budget exhausted; no action sent; use a narrower point/anchor");
|
|
6184
7501
|
if (unique && matches.Count > 1)
|
|
6185
7502
|
throw new Exception("AMBIGUOUS_UIA_TARGET: multiple controls match; no action sent. Query computer_uia, then specify hwnd and an exact id.");
|
|
6186
7503
|
if (unique && truncated)
|
|
6187
7504
|
throw new Exception("INCOMPLETE_UIA_SEARCH: uniqueness could not be verified; no action sent. Narrow the target window or query with greater depth.");
|
|
6188
7505
|
AutomationElement found = matches.Count > index ? matches[index] : null;
|
|
6189
7506
|
if (!unique) for (int i = 0; i < Math.Min(index, matches.Count); i++)
|
|
6190
|
-
try { skipped.Add(
|
|
7507
|
+
try { skipped.Add(ElemDictBudgeted(matches[i], readBudget)); }
|
|
7508
|
+
catch (UiBudgetStop) { throw new Exception("INCOMPLETE_UIA_SEARCH: provider-read budget exhausted before action; no action sent"); } catch { }
|
|
6191
7509
|
if (found == null)
|
|
6192
7510
|
throw new Exception("uiaAct: no element matched (nameContains=" + (name ?? "-") +
|
|
6193
7511
|
", automationId=" + (autoId ?? "-") + ", role=" + (role ?? "-") +
|
|
6194
7512
|
", index=" + index + ") among " + seen + " elements");
|
|
6195
7513
|
|
|
6196
7514
|
// A UIA mutation shares the same actual-action boundary and takeover checks.
|
|
6197
|
-
//
|
|
6198
|
-
//
|
|
6199
|
-
// its automation pattern either.
|
|
7515
|
+
// Scoped selectors and observed targets still need a final process
|
|
7516
|
+
// guard; a host control is never driven through its automation pattern.
|
|
6200
7517
|
int foundPid = 0;
|
|
6201
7518
|
try { foundPid = found.Current.ProcessId; } catch { }
|
|
6202
7519
|
HostGuardPid(foundPid, "uiaAct");
|
|
@@ -6529,6 +7846,9 @@ namespace DshComputerUse.Worker
|
|
|
6529
7846
|
static Dictionary<string, object> Err(Exception error)
|
|
6530
7847
|
{
|
|
6531
7848
|
var result = Err(error.Message);
|
|
7849
|
+
if (error.Data["glowHealth"] != null) result["glowHealth"] = error.Data["glowHealth"];
|
|
7850
|
+
foreach (string key in new string[] { "dragStarted", "expectedCursor", "actualCursor", "cursorCheck" })
|
|
7851
|
+
if (error.Data.Contains(key)) result[key] = error.Data[key];
|
|
6532
7852
|
if (error.Data["code"] != null)
|
|
6533
7853
|
{
|
|
6534
7854
|
result["code"] = (string)error.Data["code"];
|
|
@@ -6622,3 +7942,742 @@ namespace DshComputerUse.Worker
|
|
|
6622
7942
|
}
|
|
6623
7943
|
}
|
|
6624
7944
|
}
|
|
7945
|
+
|
|
7946
|
+
namespace DshComputerUse.Worker
|
|
7947
|
+
{
|
|
7948
|
+
/// A small, capture-excluded visual replacement at the native pointer hotspot.
|
|
7949
|
+
/// Native cursor resources remain intact for hit evidence and screenshot burn-in.
|
|
7950
|
+
/// Update/Hide/Dispose are called only by Glow's window-owning render thread.
|
|
7951
|
+
public static class AgentPointer
|
|
7952
|
+
{
|
|
7953
|
+
const int LogicalWidth = KitePointerPainter.LogicalWidth, LogicalHeight = KitePointerPainter.LogicalHeight;
|
|
7954
|
+
const int WS_POPUP = unchecked((int)0x80000000), WS_DISABLED = 0x08000000;
|
|
7955
|
+
const int CompanionStyle = 0x00080000 | 0x00000020 | 0x00000080 | 0x08000000 | 0x00000008;
|
|
7956
|
+
[StructLayout(LayoutKind.Sequential)] struct Pt { public int X, Y; }
|
|
7957
|
+
[StructLayout(LayoutKind.Sequential)] struct Sz { public int X, Y; }
|
|
7958
|
+
[StructLayout(LayoutKind.Sequential)] struct Rc { public int Left, Top, Right, Bottom; }
|
|
7959
|
+
[StructLayout(LayoutKind.Sequential)] struct MonitorInfo { public uint Size; public Rc Monitor, Work; public uint Flags; }
|
|
7960
|
+
[StructLayout(LayoutKind.Sequential)] struct CursorInfo { public uint Size, Flags; public IntPtr Cursor; public Pt Position; }
|
|
7961
|
+
[StructLayout(LayoutKind.Sequential, Pack = 1)] struct Blend { public byte Op, Flags, Alpha, Format; }
|
|
7962
|
+
[StructLayout(LayoutKind.Sequential)] struct BitmapHeader
|
|
7963
|
+
{
|
|
7964
|
+
public uint Size; public int Width, Height; public ushort Planes, BitCount;
|
|
7965
|
+
public uint Compression, SizeImage; public int XPelsPerMeter, YPelsPerMeter;
|
|
7966
|
+
public uint ClrUsed, ClrImportant;
|
|
7967
|
+
}
|
|
7968
|
+
[StructLayout(LayoutKind.Sequential)] struct BitmapInfo { public BitmapHeader Header; public uint Color; }
|
|
7969
|
+
[StructLayout(LayoutKind.Sequential)] struct Msg
|
|
7970
|
+
{
|
|
7971
|
+
public IntPtr Hwnd; public uint Message; public UIntPtr WParam;
|
|
7972
|
+
public IntPtr LParam; public uint Time; public Pt Point; public uint Private;
|
|
7973
|
+
}
|
|
7974
|
+
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
|
7975
|
+
static extern IntPtr CreateWindowEx(int ex, string cls, string title, int style, int x, int y, int w, int h, IntPtr parent, IntPtr menu, IntPtr instance, IntPtr param);
|
|
7976
|
+
[DllImport("user32.dll")] static extern bool DestroyWindow(IntPtr hwnd);
|
|
7977
|
+
[DllImport("user32.dll")] static extern bool ShowWindow(IntPtr hwnd, int command);
|
|
7978
|
+
[DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hwnd);
|
|
7979
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool SetWindowPos(IntPtr hwnd, IntPtr after, int x, int y, int w, int h, uint flags);
|
|
7980
|
+
[DllImport("user32.dll")] static extern bool GetCursorInfo(ref CursorInfo info);
|
|
7981
|
+
[DllImport("user32.dll")] static extern IntPtr MonitorFromPoint(Pt point, uint flags);
|
|
7982
|
+
[DllImport("user32.dll")] static extern bool GetMonitorInfo(IntPtr monitor, ref MonitorInfo info);
|
|
7983
|
+
[DllImport("shcore.dll")] static extern int GetDpiForMonitor(IntPtr monitor, int kind, out uint x, out uint y);
|
|
7984
|
+
[DllImport("user32.dll")] static extern bool SetWindowDisplayAffinity(IntPtr hwnd, uint affinity);
|
|
7985
|
+
[DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr hwnd);
|
|
7986
|
+
[DllImport("user32.dll")] static extern int ReleaseDC(IntPtr hwnd, IntPtr dc);
|
|
7987
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern bool PeekMessage(out Msg msg, IntPtr hwnd, uint min, uint max, uint remove);
|
|
7988
|
+
[DllImport("user32.dll")] static extern bool ValidateRect(IntPtr hwnd, IntPtr rect);
|
|
7989
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern IntPtr DispatchMessage(ref Msg msg);
|
|
7990
|
+
[DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr dc);
|
|
7991
|
+
[DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr dc);
|
|
7992
|
+
[DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr dc, IntPtr value);
|
|
7993
|
+
[DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr value);
|
|
7994
|
+
[DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr dc, ref BitmapInfo info, uint use, out IntPtr bits, IntPtr section, uint offset);
|
|
7995
|
+
[DllImport("user32.dll", SetLastError = true)] static extern bool UpdateLayeredWindow(IntPtr hwnd, IntPtr dc, ref Pt dest, ref Sz size, IntPtr source, ref Pt origin, int key, ref Blend blend, int flags);
|
|
7996
|
+
|
|
7997
|
+
static IntPtr window, screenDc, memoryDc, dib, oldBitmap, pixels, monitor;
|
|
7998
|
+
static int width, height, lastX = int.MinValue, lastY = int.MinValue, lastFrame = -1;
|
|
7999
|
+
static bool visible, failed, lastHolding;
|
|
8000
|
+
static string lastShape;
|
|
8001
|
+
static float scale = 1;
|
|
8002
|
+
static Rc monitorBounds;
|
|
8003
|
+
static byte[] copy;
|
|
8004
|
+
public static string Fault { get; private set; }
|
|
8005
|
+
|
|
8006
|
+
// Only a protected hold animates. Sixteen brightness steps over 2.5 seconds
|
|
8007
|
+
// avoid a repaint on every Glow tick; active input has a static bitmap.
|
|
8008
|
+
public static int AnimationFrame(bool holding, int tick)
|
|
8009
|
+
{
|
|
8010
|
+
return KitePointerPainter.AnimationFrame(holding, tick);
|
|
8011
|
+
}
|
|
8012
|
+
|
|
8013
|
+
public static void Update(bool show, bool holding)
|
|
8014
|
+
{
|
|
8015
|
+
if (!show || failed) { Hide(); return; }
|
|
8016
|
+
try
|
|
8017
|
+
{
|
|
8018
|
+
CursorInfo cursor = new CursorInfo(); cursor.Size = (uint)Marshal.SizeOf(typeof(CursorInfo));
|
|
8019
|
+
// Follow native cursor visibility, including text-entry apps which hide it.
|
|
8020
|
+
if (!GetCursorInfo(ref cursor) || (cursor.Flags & 1) == 0) { Hide(); return; }
|
|
8021
|
+
IntPtr nextMonitor = MonitorFromPoint(cursor.Position, 2);
|
|
8022
|
+
if (nextMonitor != monitor || window == IntPtr.Zero)
|
|
8023
|
+
{
|
|
8024
|
+
MonitorInfo info = new MonitorInfo(); info.Size = (uint)Marshal.SizeOf(typeof(MonitorInfo));
|
|
8025
|
+
if (!GetMonitorInfo(nextMonitor, ref info)) { Hide(); return; }
|
|
8026
|
+
monitorBounds = info.Monitor; monitor = nextMonitor;
|
|
8027
|
+
uint dpiX = 96, dpiY = 96;
|
|
8028
|
+
try { if (GetDpiForMonitor(monitor, 0, out dpiX, out dpiY) != 0) dpiX = 96; }
|
|
8029
|
+
catch (EntryPointNotFoundException) { dpiX = 96; }
|
|
8030
|
+
catch (DllNotFoundException) { dpiX = 96; }
|
|
8031
|
+
scale = Math.Max(1.0f, Math.Min(4.0f, dpiX / 96.0f));
|
|
8032
|
+
EnsureSurface((int)Math.Ceiling(LogicalWidth * scale), (int)Math.Ceiling(LogicalHeight * scale));
|
|
8033
|
+
lastFrame = -1;
|
|
8034
|
+
}
|
|
8035
|
+
// The drawn tip IS the native click point. Never flip or clamp the
|
|
8036
|
+
// surface near an edge: that would lie about the actual hotspot.
|
|
8037
|
+
System.Drawing.Point hotspot = KitePointerPainter.Hotspot(scale);
|
|
8038
|
+
int x = cursor.Position.X - hotspot.X, y = cursor.Position.Y - hotspot.Y;
|
|
8039
|
+
string shape = Program.StandardCursorShape(cursor.Cursor,
|
|
8040
|
+
delegate(int id) { return Native.LoadCursor(IntPtr.Zero, new IntPtr(id)); });
|
|
8041
|
+
int frame = AnimationFrame(holding, Environment.TickCount);
|
|
8042
|
+
if (frame != lastFrame || holding != lastHolding || shape != lastShape)
|
|
8043
|
+
{
|
|
8044
|
+
using (Bitmap bitmap = RenderPreview(scale, holding, frame, shape))
|
|
8045
|
+
{
|
|
8046
|
+
BitmapData data = bitmap.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, PixelFormat.Format32bppPArgb);
|
|
8047
|
+
try
|
|
8048
|
+
{
|
|
8049
|
+
if (copy == null || copy.Length != width * height * 4) copy = new byte[width * height * 4];
|
|
8050
|
+
for (int row = 0; row < height; row++)
|
|
8051
|
+
Marshal.Copy(IntPtr.Add(data.Scan0, row * data.Stride), copy, row * width * 4, width * 4);
|
|
8052
|
+
Marshal.Copy(copy, 0, pixels, copy.Length);
|
|
8053
|
+
}
|
|
8054
|
+
finally { bitmap.UnlockBits(data); }
|
|
8055
|
+
}
|
|
8056
|
+
Pt target = new Pt { X = x, Y = y }, origin = new Pt();
|
|
8057
|
+
Sz size = new Sz { X = width, Y = height };
|
|
8058
|
+
Blend blend = new Blend { Alpha = 255, Format = 1 };
|
|
8059
|
+
if (!UpdateLayeredWindow(window, screenDc, ref target, ref size, memoryDc, ref origin, 0, ref blend, 2))
|
|
8060
|
+
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8061
|
+
lastFrame = frame; lastHolding = holding; lastShape = shape; lastX = x; lastY = y;
|
|
8062
|
+
}
|
|
8063
|
+
else if (x != lastX || y != lastY)
|
|
8064
|
+
{
|
|
8065
|
+
// Moving a cached tiny layered surface does not redraw the desktop.
|
|
8066
|
+
if (!SetWindowPos(window, new IntPtr(-1), x, y, 0, 0, 0x0001 | 0x0010))
|
|
8067
|
+
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8068
|
+
lastX = x; lastY = y;
|
|
8069
|
+
}
|
|
8070
|
+
if (!visible) { ShowWindow(window, 4); visible = true; }
|
|
8071
|
+
if (!IsWindowVisible(window)) throw new InvalidOperationException("replacement pointer surface is not visible");
|
|
8072
|
+
// Publish a short hide lease only AFTER a successful visible paint.
|
|
8073
|
+
// While the independent guardian starts, retain the native cursor.
|
|
8074
|
+
if (!RequestReplacement(true))
|
|
8075
|
+
{
|
|
8076
|
+
ShowWindow(window, 0); visible = false;
|
|
8077
|
+
if (PointerVisibilityLease.Fault != null) throw new InvalidOperationException(PointerVisibilityLease.Fault);
|
|
8078
|
+
return;
|
|
8079
|
+
}
|
|
8080
|
+
Msg msg;
|
|
8081
|
+
for (int n = 0; n < 8 && PeekMessage(out msg, window, 0, 0, 1); n++)
|
|
8082
|
+
{
|
|
8083
|
+
if (msg.Message == 0x000F) ValidateRect(window, IntPtr.Zero);
|
|
8084
|
+
else DispatchMessage(ref msg);
|
|
8085
|
+
}
|
|
8086
|
+
}
|
|
8087
|
+
catch (Exception error)
|
|
8088
|
+
{
|
|
8089
|
+
// This presentation is optional. A display failure must never block input,
|
|
8090
|
+
// leak a global cursor, or turn a recoverable Glow frame into a fatal one.
|
|
8091
|
+
Fault = error.GetType().Name + ": " + error.Message; failed = true; Dispose();
|
|
8092
|
+
}
|
|
8093
|
+
}
|
|
8094
|
+
|
|
8095
|
+
static void EnsureSurface(int w, int h)
|
|
8096
|
+
{
|
|
8097
|
+
if (window == IntPtr.Zero)
|
|
8098
|
+
{
|
|
8099
|
+
window = CreateWindowEx(CompanionStyle, "STATIC", "DSH kite pointer", WS_POPUP | WS_DISABLED,
|
|
8100
|
+
0, 0, w, h, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
|
|
8101
|
+
if (window == IntPtr.Zero) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
|
|
8102
|
+
// If exclusion is unavailable, omit the decoration rather than paint it
|
|
8103
|
+
// into evidence. WS_DISABLED + TRANSPARENT prevent input interception.
|
|
8104
|
+
if (!SetWindowDisplayAffinity(window, 0x11)) throw new InvalidOperationException("pointer capture exclusion unavailable");
|
|
8105
|
+
screenDc = GetDC(IntPtr.Zero); memoryDc = CreateCompatibleDC(screenDc);
|
|
8106
|
+
if (screenDc == IntPtr.Zero || memoryDc == IntPtr.Zero) throw new InvalidOperationException("pointer DC unavailable");
|
|
8107
|
+
}
|
|
8108
|
+
if (width == w && height == h && dib != IntPtr.Zero) return;
|
|
8109
|
+
ReleaseSurface();
|
|
8110
|
+
BitmapInfo info = new BitmapInfo(); info.Header.Size = (uint)Marshal.SizeOf(typeof(BitmapHeader));
|
|
8111
|
+
info.Header.Width = w; info.Header.Height = -h; info.Header.Planes = 1; info.Header.BitCount = 32;
|
|
8112
|
+
dib = CreateDIBSection(screenDc, ref info, 0, out pixels, IntPtr.Zero, 0);
|
|
8113
|
+
if (dib == IntPtr.Zero) throw new InvalidOperationException("pointer surface unavailable");
|
|
8114
|
+
oldBitmap = SelectObject(memoryDc, dib);
|
|
8115
|
+
if (oldBitmap == IntPtr.Zero || oldBitmap == new IntPtr(-1)) throw new InvalidOperationException("pointer bitmap unavailable");
|
|
8116
|
+
width = w; height = h;
|
|
8117
|
+
}
|
|
8118
|
+
|
|
8119
|
+
// Offscreen previews and the live pointer share the exact painter/hotspot.
|
|
8120
|
+
public static Bitmap RenderPreview(float dpiScale, bool holding, int frame, string semantic = "arrow")
|
|
8121
|
+
{ return KitePointerPainter.RenderPreview(dpiScale, holding, frame, semantic); }
|
|
8122
|
+
|
|
8123
|
+
// A narrow seam for private-desktop tests; production uses the guardian lease.
|
|
8124
|
+
static bool RequestReplacement(bool show)
|
|
8125
|
+
{ if (show) return PointerVisibilityLease.TryHide(); PointerVisibilityLease.Show(); return false; }
|
|
8126
|
+
|
|
8127
|
+
public static void Hide()
|
|
8128
|
+
{
|
|
8129
|
+
RequestReplacement(false);
|
|
8130
|
+
if (window != IntPtr.Zero && visible) ShowWindow(window, 0);
|
|
8131
|
+
visible = false;
|
|
8132
|
+
}
|
|
8133
|
+
static void ReleaseSurface()
|
|
8134
|
+
{
|
|
8135
|
+
if (oldBitmap != IntPtr.Zero && oldBitmap != new IntPtr(-1) && memoryDc != IntPtr.Zero) SelectObject(memoryDc, oldBitmap);
|
|
8136
|
+
oldBitmap = IntPtr.Zero;
|
|
8137
|
+
if (dib != IntPtr.Zero) DeleteObject(dib);
|
|
8138
|
+
dib = IntPtr.Zero; pixels = IntPtr.Zero; width = height = 0; copy = null;
|
|
8139
|
+
}
|
|
8140
|
+
public static void Dispose()
|
|
8141
|
+
{
|
|
8142
|
+
Hide(); ReleaseSurface();
|
|
8143
|
+
PointerVisibilityLease.Dispose();
|
|
8144
|
+
if (memoryDc != IntPtr.Zero) DeleteDC(memoryDc);
|
|
8145
|
+
if (screenDc != IntPtr.Zero) ReleaseDC(IntPtr.Zero, screenDc);
|
|
8146
|
+
if (window != IntPtr.Zero) DestroyWindow(window);
|
|
8147
|
+
memoryDc = screenDc = window = monitor = IntPtr.Zero;
|
|
8148
|
+
lastFrame = -1; lastX = lastY = int.MinValue;
|
|
8149
|
+
}
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
8152
|
+
|
|
8153
|
+
namespace DshComputerUse.Worker
|
|
8154
|
+
{
|
|
8155
|
+
public static class KitePointerPainter
|
|
8156
|
+
{
|
|
8157
|
+
public const int LogicalWidth = 39, LogicalHeight = 44;
|
|
8158
|
+
public const float BodyScale = 0.75f, TiltDegrees = -25f;
|
|
8159
|
+
public static readonly PointF LogicalHotspot = new PointF(8, 4);
|
|
8160
|
+
static readonly double Angle = TiltDegrees * Math.PI / 180.0;
|
|
8161
|
+
|
|
8162
|
+
static float Scale(float value)
|
|
8163
|
+
{
|
|
8164
|
+
if (float.IsNaN(value) || float.IsInfinity(value)) return 1f;
|
|
8165
|
+
return Math.Max(1f, Math.Min(4f, value));
|
|
8166
|
+
}
|
|
8167
|
+
|
|
8168
|
+
public static Size CanvasSize(float dpiScale)
|
|
8169
|
+
{
|
|
8170
|
+
float s = Scale(dpiScale);
|
|
8171
|
+
return new Size((int)Math.Ceiling(LogicalWidth * s), (int)Math.Ceiling(LogicalHeight * s));
|
|
8172
|
+
}
|
|
8173
|
+
|
|
8174
|
+
public static Point Hotspot(float dpiScale)
|
|
8175
|
+
{
|
|
8176
|
+
float s = Scale(dpiScale);
|
|
8177
|
+
return new Point((int)Math.Round(LogicalHotspot.X * s, MidpointRounding.AwayFromZero),
|
|
8178
|
+
(int)Math.Round(LogicalHotspot.Y * s, MidpointRounding.AwayFromZero));
|
|
8179
|
+
}
|
|
8180
|
+
|
|
8181
|
+
// Source kite tip is (27,7). Rotation is counterclockwise in screen coordinates:
|
|
8182
|
+
// an upward-pointing head tilts toward the upper-left. Device-pixel H is shared
|
|
8183
|
+
// by the painter and the future cursor hotspot; it never depends on animation.
|
|
8184
|
+
public static PointF Transform(float x, float y, float dpiScale)
|
|
8185
|
+
{
|
|
8186
|
+
float s = Scale(dpiScale);
|
|
8187
|
+
Point h = Hotspot(s);
|
|
8188
|
+
double dx = x - 27, dy = y - 7;
|
|
8189
|
+
return new PointF(h.X + (float)(s * BodyScale * (Math.Cos(Angle) * dx - Math.Sin(Angle) * dy)),
|
|
8190
|
+
h.Y + (float)(s * BodyScale * (Math.Sin(Angle) * dx + Math.Cos(Angle) * dy)));
|
|
8191
|
+
}
|
|
8192
|
+
|
|
8193
|
+
public static int AnimationFrame(bool holding, int tick)
|
|
8194
|
+
{
|
|
8195
|
+
if (!holding) return 16;
|
|
8196
|
+
double phase = (unchecked((uint)tick) % 2500) * 2 * Math.PI / 2500.0;
|
|
8197
|
+
return 7 + (int)Math.Round(4 * (1 - Math.Cos(phase)));
|
|
8198
|
+
}
|
|
8199
|
+
|
|
8200
|
+
static PointF[] Shape(float s, params float[] xy)
|
|
8201
|
+
{
|
|
8202
|
+
PointF[] points = new PointF[xy.Length / 2];
|
|
8203
|
+
for (int i = 0; i < points.Length; i++) points[i] = Transform(xy[2 * i], xy[2 * i + 1], s);
|
|
8204
|
+
return points;
|
|
8205
|
+
}
|
|
8206
|
+
|
|
8207
|
+
public static Bitmap RenderPreview(float dpiScale, bool holding, int frame, string semantic = "arrow")
|
|
8208
|
+
{
|
|
8209
|
+
float s = Scale(dpiScale);
|
|
8210
|
+
Size size = CanvasSize(s);
|
|
8211
|
+
Bitmap bitmap = new Bitmap(size.Width, size.Height, PixelFormat.Format32bppPArgb);
|
|
8212
|
+
float pulse = holding ? Math.Max(0f, Math.Min(1f, (frame - 7) / 8f)) : 1f;
|
|
8213
|
+
int bodyAlpha = holding ? 208 + (int)(36 * pulse) : 248;
|
|
8214
|
+
PointF[] kite = Shape(s, 27, 7, 41, 24, 27, 43, 13, 24);
|
|
8215
|
+
using (Graphics g = Graphics.FromImage(bitmap))
|
|
8216
|
+
{
|
|
8217
|
+
g.Clear(Color.Transparent);
|
|
8218
|
+
g.SmoothingMode = SmoothingMode.AntiAlias;
|
|
8219
|
+
g.PixelOffsetMode = PixelOffsetMode.HighQuality;
|
|
8220
|
+
|
|
8221
|
+
// A slim visible string attaches to the lower point. No satellite star,
|
|
8222
|
+
// duplicate arrow, end bead or external status symbol crowds the tip.
|
|
8223
|
+
using (Pen tail = new Pen(Color.FromArgb(holding ? 185 + (int)(35 * pulse) : 230, 39, 118, 224), 1.1f * s))
|
|
8224
|
+
{
|
|
8225
|
+
tail.StartCap = tail.EndCap = LineCap.Round;
|
|
8226
|
+
g.DrawBezier(tail, Transform(27, 43, s), Transform(25, 53, s),
|
|
8227
|
+
Transform(11, 41, s), Transform(10, 53, s));
|
|
8228
|
+
}
|
|
8229
|
+
|
|
8230
|
+
using (Pen halo = new Pen(Color.FromArgb(holding ? 8 + (int)(20 * pulse) : 28, 54, 132, 255), 2.5f * s))
|
|
8231
|
+
g.DrawPolygon(halo, kite);
|
|
8232
|
+
using (LinearGradientBrush fill = new LinearGradientBrush(Transform(22, 9, s), Transform(33, 43, s),
|
|
8233
|
+
Color.FromArgb(bodyAlpha, holding ? 49 : 57, holding ? 133 : 150, 250),
|
|
8234
|
+
Color.FromArgb(bodyAlpha, 20, 51, holding ? 128 : 151)))
|
|
8235
|
+
g.FillPolygon(fill, kite);
|
|
8236
|
+
using (SolidBrush facet = new SolidBrush(Color.FromArgb(holding ? 78 + (int)(25 * pulse) : 115, 160, 225, 255)))
|
|
8237
|
+
g.FillPolygon(facet, Shape(s, 27, 7, 41, 24, 27, 25));
|
|
8238
|
+
using (SolidBrush shade = new SolidBrush(Color.FromArgb(78, 6, 24, 79)))
|
|
8239
|
+
g.FillPolygon(shade, Shape(s, 27, 25, 41, 24, 27, 43));
|
|
8240
|
+
using (Pen rim = new Pen(Color.FromArgb(242, 191, 231, 255), Math.Max(0.85f, 0.85f * s)))
|
|
8241
|
+
{
|
|
8242
|
+
rim.LineJoin = LineJoin.Miter;
|
|
8243
|
+
g.DrawPolygon(rim, kite);
|
|
8244
|
+
}
|
|
8245
|
+
using (Pen spine = new Pen(Color.FromArgb(72, 213, 242, 255), 0.55f * s))
|
|
8246
|
+
g.DrawLine(spine, Transform(27, 8, s), Transform(27, 42, s));
|
|
8247
|
+
|
|
8248
|
+
// The state mark stays legible at the dimmest hold frame. Geometry and
|
|
8249
|
+
// hotspot remain stationary; only light/alpha changes during breathing.
|
|
8250
|
+
if (NormalizeSemantic(semantic) == "arrow")
|
|
8251
|
+
using (SolidBrush mark = new SolidBrush(Color.FromArgb(245, 236, 249, 255)))
|
|
8252
|
+
{
|
|
8253
|
+
if (holding)
|
|
8254
|
+
{
|
|
8255
|
+
g.FillPolygon(mark, Shape(s, 23.3f, 20.2f, 25.2f, 20.2f, 25.2f, 28.2f, 23.3f, 28.2f));
|
|
8256
|
+
g.FillPolygon(mark, Shape(s, 29.1f, 20.2f, 31.0f, 20.2f, 31.0f, 28.2f, 29.1f, 28.2f));
|
|
8257
|
+
}
|
|
8258
|
+
else g.FillPolygon(mark, Shape(s, 24, 19.8f, 31.4f, 24.2f, 24, 28.6f));
|
|
8259
|
+
}
|
|
8260
|
+
DrawSemantic(g, s, semantic);
|
|
8261
|
+
}
|
|
8262
|
+
return bitmap;
|
|
8263
|
+
}
|
|
8264
|
+
|
|
8265
|
+
public static string NormalizeSemantic(string semantic)
|
|
8266
|
+
{
|
|
8267
|
+
if (String.IsNullOrEmpty(semantic)) return "arrow";
|
|
8268
|
+
string value = semantic.Trim().ToLowerInvariant();
|
|
8269
|
+
if (value == "ibeam" || value == "i-beam") return "text";
|
|
8270
|
+
switch (value)
|
|
8271
|
+
{
|
|
8272
|
+
case "arrow": case "text": case "resize-ew": case "resize-ns":
|
|
8273
|
+
case "resize-nwse": case "resize-nesw": case "move": case "hand":
|
|
8274
|
+
case "cross": case "wait": case "app-starting": case "not-allowed": case "help":
|
|
8275
|
+
return value;
|
|
8276
|
+
default: return "unknown";
|
|
8277
|
+
}
|
|
8278
|
+
}
|
|
8279
|
+
|
|
8280
|
+
// Every semantic is a small upright center mark. Neither the canvas, kite,
|
|
8281
|
+
// tail nor hotspot moves; holding keeps the same mark and only breathes.
|
|
8282
|
+
// An unknown native cursor gets a neutral dash, never a guessed resize/text mark.
|
|
8283
|
+
static void DrawSemantic(Graphics g, float s, string semantic)
|
|
8284
|
+
{
|
|
8285
|
+
semantic = NormalizeSemantic(semantic);
|
|
8286
|
+
if (semantic == "arrow") return;
|
|
8287
|
+
PointF center = Transform(27, 25, s);
|
|
8288
|
+
using (GraphicsPath path = new GraphicsPath())
|
|
8289
|
+
using (Matrix matrix = new Matrix(s, 0, 0, s, center.X, center.Y))
|
|
8290
|
+
{
|
|
8291
|
+
if (semantic == "text")
|
|
8292
|
+
{
|
|
8293
|
+
path.AddLine(-2, -4, 2, -4);
|
|
8294
|
+
path.StartFigure(); path.AddLine(0, -4, 0, 4);
|
|
8295
|
+
path.StartFigure(); path.AddLine(-2, 4, 2, 4);
|
|
8296
|
+
}
|
|
8297
|
+
else if (semantic.StartsWith("resize-"))
|
|
8298
|
+
{
|
|
8299
|
+
float angle = semantic == "resize-ns" ? 90 : semantic == "resize-nwse" ? 45 : semantic == "resize-nesw" ? -45 : 0;
|
|
8300
|
+
AddDoubleArrow(path, 4.0f, 1.9f);
|
|
8301
|
+
using (Matrix rotate = new Matrix()) { rotate.Rotate(angle); path.Transform(rotate); }
|
|
8302
|
+
}
|
|
8303
|
+
else if (semantic == "move")
|
|
8304
|
+
{
|
|
8305
|
+
AddDoubleArrow(path, 3.9f, 1.5f);
|
|
8306
|
+
using (GraphicsPath vertical = new GraphicsPath())
|
|
8307
|
+
using (Matrix rotate = new Matrix())
|
|
8308
|
+
{
|
|
8309
|
+
AddDoubleArrow(vertical, 3.9f, 1.5f); rotate.Rotate(90); vertical.Transform(rotate);
|
|
8310
|
+
path.AddPath(vertical, false);
|
|
8311
|
+
}
|
|
8312
|
+
}
|
|
8313
|
+
else if (semantic == "cross")
|
|
8314
|
+
{
|
|
8315
|
+
path.AddLine(-4, 0, 4, 0); path.StartFigure(); path.AddLine(0, -4, 0, 4);
|
|
8316
|
+
}
|
|
8317
|
+
else if (semantic == "not-allowed")
|
|
8318
|
+
{
|
|
8319
|
+
path.AddEllipse(-3.6f, -3.6f, 7.2f, 7.2f);
|
|
8320
|
+
path.StartFigure(); path.AddLine(-2.5f, 2.5f, 2.5f, -2.5f);
|
|
8321
|
+
}
|
|
8322
|
+
else if (semantic == "wait")
|
|
8323
|
+
{
|
|
8324
|
+
AddHourglass(path, 0, 0, 2.6f, 3.8f);
|
|
8325
|
+
}
|
|
8326
|
+
else if (semantic == "app-starting")
|
|
8327
|
+
{
|
|
8328
|
+
path.AddLines(new PointF[] { new PointF(-3.5f, 2.7f), new PointF(-3.5f, -3.8f), new PointF(-0.2f, -0.4f) });
|
|
8329
|
+
path.StartFigure(); AddHourglass(path, 1.8f, 1.2f, 1.5f, 2.6f);
|
|
8330
|
+
}
|
|
8331
|
+
else if (semantic == "help")
|
|
8332
|
+
{
|
|
8333
|
+
path.AddBezier(-2.4f, -2.4f, -2.0f, -5.3f, 4.1f, -4.0f, 1.7f, -1.3f);
|
|
8334
|
+
path.AddBezier(1.7f, -1.3f, 0.6f, -0.5f, 0, -0.4f, 0, 1.2f);
|
|
8335
|
+
path.StartFigure(); path.AddLine(0, 3.3f, 0, 3.6f);
|
|
8336
|
+
}
|
|
8337
|
+
else if (semantic == "hand")
|
|
8338
|
+
{
|
|
8339
|
+
path.AddPolygon(new PointF[] { new PointF(-3.7f, 0), new PointF(-2.4f, -1),
|
|
8340
|
+
new PointF(-1, 0.4f), new PointF(-1, -4.2f), new PointF(0.6f, -4.2f),
|
|
8341
|
+
new PointF(0.6f, -0.8f), new PointF(1.8f, -1.4f), new PointF(3, -0.5f),
|
|
8342
|
+
new PointF(3.7f, 0.6f), new PointF(3.1f, 3.6f), new PointF(-0.2f, 3.9f), new PointF(-2, 2) });
|
|
8343
|
+
}
|
|
8344
|
+
else
|
|
8345
|
+
{
|
|
8346
|
+
path.AddLine(-2.1f, 0, 2.1f, 0);
|
|
8347
|
+
}
|
|
8348
|
+
path.Transform(matrix);
|
|
8349
|
+
using (Pen contrast = new Pen(Color.FromArgb(110, 8, 32, 89), 2.0f * s))
|
|
8350
|
+
using (Pen ink = new Pen(Color.FromArgb(semantic == "unknown" ? 180 : 248, 232, 247, 255), 1.0f * s))
|
|
8351
|
+
{
|
|
8352
|
+
contrast.LineJoin = ink.LineJoin = LineJoin.Round;
|
|
8353
|
+
contrast.StartCap = contrast.EndCap = ink.StartCap = ink.EndCap = LineCap.Round;
|
|
8354
|
+
g.DrawPath(contrast, path); g.DrawPath(ink, path);
|
|
8355
|
+
}
|
|
8356
|
+
}
|
|
8357
|
+
}
|
|
8358
|
+
|
|
8359
|
+
static void AddDoubleArrow(GraphicsPath path, float radius, float head)
|
|
8360
|
+
{
|
|
8361
|
+
path.AddLine(-radius, 0, radius, 0); path.StartFigure();
|
|
8362
|
+
path.AddLines(new PointF[] { new PointF(-radius + head, -head), new PointF(-radius, 0), new PointF(-radius + head, head) });
|
|
8363
|
+
path.StartFigure();
|
|
8364
|
+
path.AddLines(new PointF[] { new PointF(radius - head, -head), new PointF(radius, 0), new PointF(radius - head, head) });
|
|
8365
|
+
}
|
|
8366
|
+
|
|
8367
|
+
static void AddHourglass(GraphicsPath path, float x, float y, float width, float height)
|
|
8368
|
+
{
|
|
8369
|
+
path.AddLines(new PointF[] { new PointF(x - width, y - height), new PointF(x + width, y - height),
|
|
8370
|
+
new PointF(x - width, y + height), new PointF(x + width, y + height), new PointF(x - width, y - height) });
|
|
8371
|
+
}
|
|
8372
|
+
}
|
|
8373
|
+
|
|
8374
|
+
}
|
|
8375
|
+
|
|
8376
|
+
namespace DshComputerUse.Worker
|
|
8377
|
+
{
|
|
8378
|
+
public static class PointerVisibilityLease
|
|
8379
|
+
{
|
|
8380
|
+
const int Version = 1, Bytes = 128, ReadyOffset = 48, RestoredOffset = 52;
|
|
8381
|
+
const int LeaseMs = 500, RefreshMs = 100, StartupMs = 2000;
|
|
8382
|
+
const string Prefix = "Local\\DSH-CursorLease-";
|
|
8383
|
+
static readonly object Gate = new object();
|
|
8384
|
+
static Session current;
|
|
8385
|
+
static string fault;
|
|
8386
|
+
public static string Fault { get { lock (Gate) return fault; } }
|
|
8387
|
+
internal static long NowMs()
|
|
8388
|
+
{ return (long)(System.Diagnostics.Stopwatch.GetTimestamp() * (1000.0 / System.Diagnostics.Stopwatch.Frequency)); }
|
|
8389
|
+
|
|
8390
|
+
sealed class Session
|
|
8391
|
+
{
|
|
8392
|
+
public System.IO.MemoryMappedFiles.MemoryMappedFile Map;
|
|
8393
|
+
public System.IO.MemoryMappedFiles.MemoryMappedViewAccessor View;
|
|
8394
|
+
public System.Diagnostics.Process Guardian;
|
|
8395
|
+
public bool Ready, Revoked, Requested, Finished;
|
|
8396
|
+
public long Sequence, LastRefresh;
|
|
8397
|
+
}
|
|
8398
|
+
|
|
8399
|
+
// Invoke ONLY after the overlay has been successfully presented at the
|
|
8400
|
+
// native hotspot. True means the ready guardian accepted a current lease,
|
|
8401
|
+
// not proof that pixels disappeared. Never blocks waiting for readiness.
|
|
8402
|
+
public static bool TryHide()
|
|
8403
|
+
{
|
|
8404
|
+
lock (Gate)
|
|
8405
|
+
{
|
|
8406
|
+
if (fault != null) return false;
|
|
8407
|
+
if (current != null && current.Finished) current = null;
|
|
8408
|
+
if (current == null)
|
|
8409
|
+
{
|
|
8410
|
+
current = new Session(); Session launch = current;
|
|
8411
|
+
System.Threading.ThreadPool.QueueUserWorkItem(delegate { StartAndMonitor(launch); });
|
|
8412
|
+
return false;
|
|
8413
|
+
}
|
|
8414
|
+
Session s = current;
|
|
8415
|
+
if (s.Revoked || !s.Ready || s.View == null) return false;
|
|
8416
|
+
long now = NowMs();
|
|
8417
|
+
if (!s.Requested || now - s.LastRefresh >= RefreshMs)
|
|
8418
|
+
{
|
|
8419
|
+
s.Requested = true; // Set before writing: unknown write outcome also needs cleanup.
|
|
8420
|
+
try { WriteRequest(s, true, false, now); s.LastRefresh = now; }
|
|
8421
|
+
catch (System.Exception e) { fault = "cursor lease write: " + e.Message; s.Revoked = true; return false; }
|
|
8422
|
+
}
|
|
8423
|
+
return true;
|
|
8424
|
+
}
|
|
8425
|
+
}
|
|
8426
|
+
|
|
8427
|
+
// A nonblocking release request. Guardian observes it within its next
|
|
8428
|
+
// progressing <=20 ms poll; it is not a synchronous compositor fence.
|
|
8429
|
+
public static void Show()
|
|
8430
|
+
{
|
|
8431
|
+
lock (Gate)
|
|
8432
|
+
{
|
|
8433
|
+
Session s = current;
|
|
8434
|
+
if (s == null || s.Finished || s.Revoked) return;
|
|
8435
|
+
s.Revoked = true;
|
|
8436
|
+
if (s.View != null)
|
|
8437
|
+
{
|
|
8438
|
+
try { WriteRequest(s, false, true, NowMs()); }
|
|
8439
|
+
catch (System.Exception e) { fault = "cursor lease release: " + e.Message; }
|
|
8440
|
+
}
|
|
8441
|
+
}
|
|
8442
|
+
}
|
|
8443
|
+
public static void Dispose() { Show(); }
|
|
8444
|
+
|
|
8445
|
+
static void WriteRequest(Session s, bool desired, bool revoked, long stamp)
|
|
8446
|
+
{
|
|
8447
|
+
long even = s.Sequence + 2;
|
|
8448
|
+
s.View.Write(16, even - 1); System.Threading.Thread.MemoryBarrier();
|
|
8449
|
+
s.View.Write(24, desired ? 1 : 0); s.View.Write(28, revoked ? 1 : 0); s.View.Write(32, stamp);
|
|
8450
|
+
System.Threading.Thread.MemoryBarrier(); s.View.Write(16, even); s.Sequence = even;
|
|
8451
|
+
}
|
|
8452
|
+
internal sealed class Request
|
|
8453
|
+
{
|
|
8454
|
+
public long Sequence, Stamp;
|
|
8455
|
+
public bool Desired, Revoked;
|
|
8456
|
+
public Request(long sequence, long stamp, bool desired, bool revoked)
|
|
8457
|
+
{ Sequence = sequence; Stamp = stamp; Desired = desired; Revoked = revoked; }
|
|
8458
|
+
}
|
|
8459
|
+
internal static Request ReadRequest(System.IO.MemoryMappedFiles.MemoryMappedViewAccessor view)
|
|
8460
|
+
{
|
|
8461
|
+
long first = view.ReadInt64(16);
|
|
8462
|
+
if (first <= 0 || (first & 1) != 0) return null;
|
|
8463
|
+
System.Threading.Thread.MemoryBarrier();
|
|
8464
|
+
int desired = view.ReadInt32(24), revoked = view.ReadInt32(28); long stamp = view.ReadInt64(32);
|
|
8465
|
+
System.Threading.Thread.MemoryBarrier(); long second = view.ReadInt64(16);
|
|
8466
|
+
if (first != second || (desired != 0 && desired != 1) || (revoked != 0 && revoked != 1)) return null;
|
|
8467
|
+
return new Request(second, stamp, desired == 1, revoked == 1);
|
|
8468
|
+
}
|
|
8469
|
+
|
|
8470
|
+
static void StartAndMonitor(Session s)
|
|
8471
|
+
{
|
|
8472
|
+
System.IO.MemoryMappedFiles.MemoryMappedFile map = null;
|
|
8473
|
+
System.IO.MemoryMappedFiles.MemoryMappedViewAccessor view = null;
|
|
8474
|
+
System.Diagnostics.Process child = null;
|
|
8475
|
+
bool restoreNeeded = false, exitConfirmed = false, stopAttempted = false;
|
|
8476
|
+
try
|
|
8477
|
+
{
|
|
8478
|
+
string name = Prefix + System.Guid.NewGuid().ToString("N");
|
|
8479
|
+
using (System.Diagnostics.Process owner = System.Diagnostics.Process.GetCurrentProcess())
|
|
8480
|
+
{
|
|
8481
|
+
map = System.IO.MemoryMappedFiles.MemoryMappedFile.CreateNew(name, Bytes);
|
|
8482
|
+
view = map.CreateViewAccessor(0, Bytes);
|
|
8483
|
+
view.Write(0, Version); view.Write(4, owner.Id);
|
|
8484
|
+
long start = owner.StartTime.ToUniversalTime().Ticks; view.Write(8, start);
|
|
8485
|
+
lock (Gate)
|
|
8486
|
+
{
|
|
8487
|
+
s.Map = map; s.View = view;
|
|
8488
|
+
if (s.Revoked) return; // No child and no Mag API if released during startup.
|
|
8489
|
+
WriteRequest(s, false, false, NowMs());
|
|
8490
|
+
}
|
|
8491
|
+
var info = new System.Diagnostics.ProcessStartInfo(
|
|
8492
|
+
System.Reflection.Assembly.GetExecutingAssembly().Location,
|
|
8493
|
+
"--pointer-guardian " + name + " " + owner.Id + " " + start);
|
|
8494
|
+
info.UseShellExecute = false; info.CreateNoWindow = true;
|
|
8495
|
+
info.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
|
|
8496
|
+
child = System.Diagnostics.Process.Start(info);
|
|
8497
|
+
if (child == null) throw new System.InvalidOperationException("guardian did not start");
|
|
8498
|
+
lock (Gate) s.Guardian = child;
|
|
8499
|
+
}
|
|
8500
|
+
long began = NowMs();
|
|
8501
|
+
while (!child.WaitForExit(20))
|
|
8502
|
+
{
|
|
8503
|
+
int status = view.ReadInt32(ReadyOffset);
|
|
8504
|
+
lock (Gate) s.Ready = status == 1 && !s.Revoked;
|
|
8505
|
+
if (status == 0 && NowMs() - began > StartupMs)
|
|
8506
|
+
{
|
|
8507
|
+
// Only our own Process handle is terminated. It was not
|
|
8508
|
+
// ready, so TryHide has never authorized a hide request.
|
|
8509
|
+
stopAttempted = true;
|
|
8510
|
+
exitConfirmed = ConfirmExit(delegate { return child.HasExited; }, child.Kill, child.WaitForExit);
|
|
8511
|
+
if (!exitConfirmed) throw new System.TimeoutException("guardian termination unconfirmed after 1000 ms");
|
|
8512
|
+
throw new System.TimeoutException("guardian startup timeout");
|
|
8513
|
+
}
|
|
8514
|
+
}
|
|
8515
|
+
exitConfirmed = true;
|
|
8516
|
+
lock (Gate)
|
|
8517
|
+
{
|
|
8518
|
+
s.Ready = false;
|
|
8519
|
+
restoreNeeded = s.Requested && view.ReadInt32(RestoredOffset) != 1;
|
|
8520
|
+
if (!s.Revoked) fault = "pointer guardian ended; native cursor fallback";
|
|
8521
|
+
}
|
|
8522
|
+
}
|
|
8523
|
+
catch (System.Exception e)
|
|
8524
|
+
{
|
|
8525
|
+
lock (Gate) { s.Ready = false; fault = "pointer guardian: " + e.Message; restoreNeeded = s.Requested; }
|
|
8526
|
+
// An alive guardian remains the sole hide writer. Never race an
|
|
8527
|
+
// emergency show against a process that can still issue hide.
|
|
8528
|
+
if (child != null && !exitConfirmed && !stopAttempted)
|
|
8529
|
+
{
|
|
8530
|
+
stopAttempted = true;
|
|
8531
|
+
exitConfirmed = ConfirmExit(delegate { return child.HasExited; }, child.Kill, child.WaitForExit);
|
|
8532
|
+
}
|
|
8533
|
+
if (!exitConfirmed)
|
|
8534
|
+
{
|
|
8535
|
+
restoreNeeded = false;
|
|
8536
|
+
if (child != null) lock (Gate) fault += "; guardian exit unconfirmed, emergency show suppressed and further leases disabled";
|
|
8537
|
+
}
|
|
8538
|
+
}
|
|
8539
|
+
finally
|
|
8540
|
+
{
|
|
8541
|
+
// Guardian has ended before emergency restoration and before a
|
|
8542
|
+
// subsequent session may start. Old cleanup cannot unhide a new lease.
|
|
8543
|
+
if (restoreNeeded && exitConfirmed) EmergencyShow();
|
|
8544
|
+
lock (Gate)
|
|
8545
|
+
{
|
|
8546
|
+
s.Ready = false; s.View = null; s.Map = null; s.Guardian = null;
|
|
8547
|
+
if (view != null) view.Dispose(); if (map != null) map.Dispose();
|
|
8548
|
+
if (child != null) child.Dispose(); s.Finished = true;
|
|
8549
|
+
}
|
|
8550
|
+
}
|
|
8551
|
+
}
|
|
8552
|
+
|
|
8553
|
+
// Bounded process shutdown, shared with fake process tests. Failure is
|
|
8554
|
+
// explicit: the caller must not race show against a possible late hide.
|
|
8555
|
+
internal static bool ConfirmExit(System.Func<bool> hasExited, System.Action kill, System.Func<int, bool> wait)
|
|
8556
|
+
{
|
|
8557
|
+
try
|
|
8558
|
+
{
|
|
8559
|
+
if (hasExited()) return true;
|
|
8560
|
+
try { kill(); } catch { return hasExited(); }
|
|
8561
|
+
return wait(1000) || hasExited();
|
|
8562
|
+
}
|
|
8563
|
+
catch { return false; }
|
|
8564
|
+
}
|
|
8565
|
+
|
|
8566
|
+
static void EmergencyShow()
|
|
8567
|
+
{
|
|
8568
|
+
var native = new NativeBackend(); bool initialized = false;
|
|
8569
|
+
try { initialized = native.Initialize(); if (!initialized || !native.Show(true)) lock (Gate) fault = "emergency cursor restore API failed"; }
|
|
8570
|
+
catch (System.Exception e) { lock (Gate) fault = "emergency cursor restore: " + e.Message; }
|
|
8571
|
+
finally { if (initialized) try { native.Uninitialize(); } catch { } }
|
|
8572
|
+
}
|
|
8573
|
+
|
|
8574
|
+
internal interface Backend { bool Initialize(); bool Show(bool visible); void Uninitialize(); }
|
|
8575
|
+
sealed class NativeBackend : Backend
|
|
8576
|
+
{
|
|
8577
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagInitialize();
|
|
8578
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagShowSystemCursor(bool show);
|
|
8579
|
+
[System.Runtime.InteropServices.DllImport("Magnification.dll")] static extern bool MagUninitialize();
|
|
8580
|
+
public bool Initialize() { return MagInitialize(); }
|
|
8581
|
+
public bool Show(bool visible) { return MagShowSystemCursor(visible); }
|
|
8582
|
+
public void Uninitialize() { MagUninitialize(); }
|
|
8583
|
+
}
|
|
8584
|
+
internal sealed class Outcome
|
|
8585
|
+
{ public bool HideAttempted, RestoreConfirmed, Success; public string Reason; }
|
|
8586
|
+
|
|
8587
|
+
// Same loop is used in the actual guardian and deterministic fake-backend
|
|
8588
|
+
// tests. No wall-clock waiting or native API is required in those tests.
|
|
8589
|
+
internal static Outcome RunLease(Backend api, System.Func<Request> read,
|
|
8590
|
+
System.Func<long> now, System.Func<bool> ownerAlive, System.Action pause, System.Action ready)
|
|
8591
|
+
{
|
|
8592
|
+
var result = new Outcome(); bool initialized = false, hidden = false;
|
|
8593
|
+
long started = now(), latestSequence = 0, stamp = started; bool everRequested = false;
|
|
8594
|
+
try
|
|
8595
|
+
{
|
|
8596
|
+
initialized = api.Initialize();
|
|
8597
|
+
if (!initialized) { result.Reason = "initialize-failed"; return result; }
|
|
8598
|
+
ready();
|
|
8599
|
+
while (true)
|
|
8600
|
+
{
|
|
8601
|
+
if (!ownerAlive()) { result.Reason = "owner-exited"; break; }
|
|
8602
|
+
// Expiration is terminal even when a newer request has
|
|
8603
|
+
// arrived while this process was not scheduled.
|
|
8604
|
+
if (everRequested && now() - stamp >= LeaseMs) { result.Reason = "lease-expired"; break; }
|
|
8605
|
+
Request request = read(); long currentTime = now();
|
|
8606
|
+
if (request != null && request.Sequence > latestSequence)
|
|
8607
|
+
{
|
|
8608
|
+
latestSequence = request.Sequence;
|
|
8609
|
+
if (request.Revoked || (!request.Desired && everRequested)) { result.Reason = "released"; break; }
|
|
8610
|
+
if (request.Desired)
|
|
8611
|
+
{
|
|
8612
|
+
if (request.Stamp > currentTime || currentTime - request.Stamp >= LeaseMs)
|
|
8613
|
+
{ result.Reason = "expired-request"; break; }
|
|
8614
|
+
everRequested = true; stamp = request.Stamp;
|
|
8615
|
+
if (!hidden)
|
|
8616
|
+
{
|
|
8617
|
+
// Re-read the current complete request before the
|
|
8618
|
+
// sole hide writer acts; queued old commands never apply.
|
|
8619
|
+
Request newest = read();
|
|
8620
|
+
if (newest == null || newest.Sequence != request.Sequence) { pause(); continue; }
|
|
8621
|
+
if (!ownerAlive() || now() - stamp >= LeaseMs) { result.Reason = "expired-before-hide"; break; }
|
|
8622
|
+
result.HideAttempted = true;
|
|
8623
|
+
if (!api.Show(false)) { result.Reason = "hide-failed"; break; }
|
|
8624
|
+
hidden = true;
|
|
8625
|
+
// A release concurrent with the API is processed
|
|
8626
|
+
// before sleeping or accepting another lease.
|
|
8627
|
+
Request after = read();
|
|
8628
|
+
if (after != null && after.Sequence > latestSequence && (after.Revoked || !after.Desired))
|
|
8629
|
+
{ result.Reason = "released-during-hide"; break; }
|
|
8630
|
+
}
|
|
8631
|
+
}
|
|
8632
|
+
}
|
|
8633
|
+
if (everRequested && currentTime - stamp >= LeaseMs) { result.Reason = "lease-expired"; break; }
|
|
8634
|
+
if (!everRequested && currentTime - started >= StartupMs) { result.Reason = "unused"; break; }
|
|
8635
|
+
pause();
|
|
8636
|
+
}
|
|
8637
|
+
result.Success = result.Reason != "hide-failed";
|
|
8638
|
+
}
|
|
8639
|
+
catch (System.Exception e) { result.Reason = "fault: " + e.Message; }
|
|
8640
|
+
finally
|
|
8641
|
+
{
|
|
8642
|
+
if (initialized)
|
|
8643
|
+
{
|
|
8644
|
+
if (result.HideAttempted)
|
|
8645
|
+
{
|
|
8646
|
+
try { result.RestoreConfirmed = api.Show(true); }
|
|
8647
|
+
catch { result.RestoreConfirmed = false; }
|
|
8648
|
+
if (!result.RestoreConfirmed) result.Success = false;
|
|
8649
|
+
}
|
|
8650
|
+
try { api.Uninitialize(); } catch { result.Success = false; }
|
|
8651
|
+
}
|
|
8652
|
+
}
|
|
8653
|
+
return result;
|
|
8654
|
+
}
|
|
8655
|
+
|
|
8656
|
+
// Dispatch this at the FIRST line of worker Main, before Panic.Init,
|
|
8657
|
+
// hooks, global brake, protocol stdout, or any other worker lifecycle.
|
|
8658
|
+
public static int GuardianMain(string name, int ownerPid, long ownerStartTicks)
|
|
8659
|
+
{
|
|
8660
|
+
if (System.IntPtr.Size != 8 || name == null || !name.StartsWith(Prefix, System.StringComparison.Ordinal) ||
|
|
8661
|
+
name.Length != Prefix.Length + 32) return 2;
|
|
8662
|
+
try
|
|
8663
|
+
{
|
|
8664
|
+
using (System.Diagnostics.Process owner = System.Diagnostics.Process.GetProcessById(ownerPid))
|
|
8665
|
+
using (System.Diagnostics.Process self = System.Diagnostics.Process.GetCurrentProcess())
|
|
8666
|
+
using (var map = System.IO.MemoryMappedFiles.MemoryMappedFile.OpenExisting(name))
|
|
8667
|
+
using (var view = map.CreateViewAccessor(0, Bytes))
|
|
8668
|
+
{
|
|
8669
|
+
if (owner.StartTime.ToUniversalTime().Ticks != ownerStartTicks || owner.SessionId != self.SessionId ||
|
|
8670
|
+
view.ReadInt32(0) != Version || view.ReadInt32(4) != ownerPid || view.ReadInt64(8) != ownerStartTicks) return 3;
|
|
8671
|
+
Outcome result = RunLease(new NativeBackend(), delegate { return ReadRequest(view); }, NowMs,
|
|
8672
|
+
delegate { try { return !owner.HasExited; } catch { return false; } },
|
|
8673
|
+
delegate { System.Threading.Thread.Sleep(20); },
|
|
8674
|
+
delegate { view.Write(ReadyOffset, 1); });
|
|
8675
|
+
view.Write(RestoredOffset, result.RestoreConfirmed ? 1 : 0);
|
|
8676
|
+
System.Threading.Thread.MemoryBarrier(); view.Write(ReadyOffset, result.Success ? 2 : 3);
|
|
8677
|
+
return result.Success ? 0 : 4;
|
|
8678
|
+
}
|
|
8679
|
+
}
|
|
8680
|
+
catch { return 5; }
|
|
8681
|
+
}
|
|
8682
|
+
}
|
|
8683
|
+
}
|